gfwleak/zhangjianlong-galaxy-auto-deploy-cluster
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1. 新增自动检测安装ansible(本地yum源的方式实现)

Browse Source 
2. storm替换flink(但是storm并未完全移除,可通过改变配置部署)
3. 新增dos定时任务和dos数据流
4. ck新增表, 服务镜像配置更新, druid新增任务/更新任务
This commit is contained in:
jianlong
2021-09-01 11:30:43 +08:00
parent f2dd6f8daa
commit fd1b752d16
218 changed files with 4863 additions and 7558 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
.idea/workspace.xml generated
View File

@@ -1,8 +1,184 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="55c8c5b9-9b57-431e-a0ed-0064b85979fa" name="Default Changelist" comment="">
<list default="true" id="55c8c5b9-9b57-431e-a0ed-0064b85979fa" name="Default Changelist" comment="1. 新增自动检测安装ansible(本地yum源的方式实现)&#10;2. storm替换flink(但是storm并未完全移除,可通过改变配置部署)&#10;3. 新增dos定时任务和dos数据流&#10;4. ck新增表, 服务镜像配置更新, druid新增任务/更新任务">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/install.sh" beforeDir="false" afterPath="$PROJECT_DIR$/install.sh" afterDir="false" />
<change beforePath="$PROJECT_DIR$/inventories/pro/group_vars/all" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/inventories/pro/hosts" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Arangodb.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Arangodb.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Clickhouse.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Clickhouse.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Druid.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Druid.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/HBase.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/HBase.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Hadoop.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Hadoop.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Kafka.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Kafka.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Mariadb.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Mariadb.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Nacos.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Nacos.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Spark.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Spark.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Storm.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Storm.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/Zookeeper.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/Zookeeper.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/check-components.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/check-components.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/check-services.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/check-services.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/environment.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/environment.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-chproxy.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-chproxy.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-gateway-keepalive.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-gateway-keepalive.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-gateway-nginx.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-gateway-nginx.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-gohangout.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-gohangout.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-hos-keepalive.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-hos-keepalive.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-hos-nginx.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-hos-nginx.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-hos-service.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-hos-service.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-job-service.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-job-service.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-qgw-service.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-qgw-service.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/galaxy-report-service.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/galaxy-report-service.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-clickhouse.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-clickhouse.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-druid.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-druid.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-galaxy-gateway-keepalive.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-galaxy-gateway-keepalive.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-galaxy-hos-service.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-galaxy-hos-service.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-galaxy-job-service.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-galaxy-job-service.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-hbase.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-hbase.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-kafka.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-kafka.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-spark.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-spark.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/init-storm.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/init-storm.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/menu/commonMenu" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/menu/commonMenu" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/menu/homePage" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/menu/homePage" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/check/check-components/tasks/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/check/check-components/tasks/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/check/check-services/tasks/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/check/check-services/tasks/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/arangodb/files/arangodb.tar.gz" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/arangodb/files/arangodb.tar.gz" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/arangodb/files/conf.bak.tar" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/arangodb/vars/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/arangodb/vars/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/clickhouse/common/tasks/install_clickhouse.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/clickhouse/common/tasks/install_clickhouse.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/clickhouse/common/templates/config.xml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/clickhouse/common/templates/config.xml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/clickhouse/common/templates/start_exporter.sh.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/clickhouse/common/templates/start_exporter.sh.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/clickhouse/common/vars/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/clickhouse/common/vars/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/clickhouse/data/tasks/install_clickhouse.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/clickhouse/data/tasks/install_clickhouse.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/clickhouse/data/vars/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/clickhouse/data/vars/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/clickhouse/query/vars/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/clickhouse/query/vars/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/druid/tasks/install_druid.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/druid/tasks/install_druid.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/druid/templates/keepdruidall.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/druid/templates/keepdruidall.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/hadoop/tasks/install_hadoop.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/hadoop/tasks/install_hadoop.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/components/mariadb/install-mariadb-cluster/tasks/install-mariadb.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/components/mariadb/install-mariadb-cluster/tasks/install-mariadb.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/clickhouse/templates/create_ck_table.sql" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/clickhouse/templates/create_ck_table.sql" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/clickhouse/templates/create_ck_table.sql_21.06" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/clickhouse/templates/dll.sql" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_segments-tsg-21.05.sql" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_segments-tsg3.0.sql" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_segments-tsg3.0.sql" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/initData.sh" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_client_internal_ip_hot_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_hits_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_urls_hot_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_hits_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_urls_hot_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_server_external_ip_hot_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_sys_storage_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_client_ip_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_external_host_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_internal_host_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_server_ip_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_urls_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_user_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_website_domain_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_app_stat_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_metrics_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_protocol_stat_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_summary_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_user_hot_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/init_website_domain_hot_log.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/initDruidData/tasks.txt" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/rule/common_task.txt" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/rule/common_task.txt" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/tasks/traffic_summary_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/tasks/traffic_summary_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/topology_list.txt" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/druid/files/druid_topology/topology_list.txt" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments.zip" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments.zip" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/client_internal_ip_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_39.273Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/proxy_event_hits_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_41.641Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/proxy_event_urls_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_43.849Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/security_event_hits_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_46.392Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/security_event_urls_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_48.458Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/server_external_ip_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_50.092Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_52.202Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/top_client_ip_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_53.675Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/top_external_host_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_55.845Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/top_internal_host_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_58.433Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/top_server_ip_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_59.654Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/top_urls_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_01.484Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/top_user_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_04.063Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/top_website_domain_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_06.100Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/traffic_app_stat_log/30000101T000000.000Z_30000102T000000.000Z/2021-07-02T02_43_09.615Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/traffic_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_08.462Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/traffic_protocol_stat_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_09.706Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/traffic_summary_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_12.193Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/user_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_14.350Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/files/segments/website_domain_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_16.425Z/0_index.zip" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/druid/templates/ingestionTask.sh.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/druid/templates/ingestionTask.sh.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/galaxy-job-service/templates/xxl_job.sql.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/galaxy-job-service/templates/xxl_job.sql.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/hbase/templates/create-hbase-table.sh.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/hbase/templates/create-hbase-table.sh.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/kafka/templates/create_topic.sh.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/kafka/templates/create_topic.sh.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/spark/templates/application.properties.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/spark/templates/application.properties.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/ACTIVE-DEFENCE-EVENT-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/ACTIVE-DEFENCE-EVENT-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/CONNECTION-RECORD-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/CONNECTION-RECORD-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/GTPC-RECORD-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/GTPC-RECORD-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/LIVE-CHARTS_CONNECTION-RECORD-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/LIVE-CHARTS_CONNECTION-RECORD-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/LIVE-CHARTS_LIVE-SESSION-RECORD-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/LIVE-CHARTS_LIVE-SESSION-RECORD-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/LIVE-SESSION-RECORD-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/LIVE-SESSION-RECORD-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/PROXY-EVENT-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/PROXY-EVENT-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/RADIUS-ON-OFF.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/RADIUS-ON-OFF.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/RADIUS-RECORD-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/RADIUS-RECORD-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/SECURITY-EVENT-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/SECURITY-EVENT-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/SUBSCRUBER-HBASE.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/SUBSCRUBER-HBASE.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/SYS-PACKET-CAPTURE-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/SYS-PACKET-CAPTURE-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/TRANSACTION-RECORD-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/TRANSACTION-RECORD-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/VOIP-RECORD-COMPLETED-LOG.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/VOIP-RECORD-COMPLETED-LOG.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/init/storm/templates/VOIP-RELATION.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/init/storm/templates/VOIP-RELATION.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-chproxy/templates/config.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-chproxy/templates/config.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gateway-keepalive/tasks/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gateway-keepalive/tasks/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gateway-keepalive/templates/kp_daemon.sh.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gateway-keepalive/templates/kp_daemon.sh.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/files/start_all.sh" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/files/start_all.sh" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/tasks/load_gohangout.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/tasks/load_gohangout.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_active_defence_event_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_active_defence_event_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_connection_record_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_connection_record_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_gtpc_record_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_gtpc_record_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_live_session_record_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_live_session_record_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_proxy_event_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_proxy_event_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_onff_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_onff_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_record_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_record_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_security_event_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_security_event_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_sys_packet_capture_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_sys_packet_capture_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_transaction_record_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_transaction_record_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_voip_record_log_tsgv3.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-gohangout/templates/k2ck_voip_record_log_tsgv3.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/templates/application.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/templates/application.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/templates/galaxy-hos-service.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/templates/galaxy-hos-service.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/templates/log4j2-dev.xml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/templates/log4j2-dev.xml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/templates/tmp/application.properties.j2" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/templates/tmp/galaxy-hos-service.j2" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/vars/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-hos-service/vars/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-job-service/templates/galaxy-job-executor.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-job-service/templates/galaxy-job-executor.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-job-service/templates/xxl_job.sql.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-job-service/templates/xxl_job.sql.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-job-service/vars/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-job-service/vars/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/ck-filter.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/ck-filter.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/ck-queries-template.sql" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/ck-queries-template.sql" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_client_ip.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_client_ip.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_server_ip.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_server_ip.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_subscriber_id.json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_http_domain.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_http_domain.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/druid-filter.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/druid-filter.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/engine-filter.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/engine-filter.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/gtpc_record_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/gtpc_record_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_history.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_history.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_live.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_live.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/live_session_record_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/live_session_record_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/meta_data.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/meta_data.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/proxy_event_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/proxy_event_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/radius_record_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/radius_record_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/security_event_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/security_event_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/sys_packet_capture_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/sys_packet_capture_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/transaction_record_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/transaction_record_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/version.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/version.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/voip_record_log.json" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/files/config/voip_record_log.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/vars/main.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/roles/services/galaxy-qgw-service/vars/main.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/test.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/test.yml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/parcels/test1.yml" beforeDir="false" afterPath="$PROJECT_DIR$/parcels/test1.yml" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
@@ -15,20 +191,8 @@
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/install.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="330">
<caret line="22" column="10" lean-forward="true" selection-start-line="22" selection-start-column="10" selection-end-line="22" selection-end-column="10" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="FIRST">
<first_editor relative-caret-position="960">
<caret line="64" column="27" selection-start-line="64" selection-start-column="27" selection-end-line="64" selection-end-column="27" />
</first_editor>
<second_editor />
<state relative-caret-position="689">
<caret line="53" column="22" lean-forward="true" selection-start-line="53" selection-start-column="22" selection-end-line="53" selection-end-column="22" />
</state>
</provider>
</entry>
@@ -65,6 +229,11 @@
<item name="Galaxy-auto-deploy-cluster" type="b2602c69:ProjectViewProjectNode" />
<item name="Galaxy-auto-deploy-cluster" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="Galaxy-auto-deploy-cluster" type="b2602c69:ProjectViewProjectNode" />
<item name="Galaxy-auto-deploy-cluster" type="462c0819:PsiDirectoryNode" />
<item name="parcels" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
@@ -104,17 +273,28 @@
<workItem from="1629364020595" duration="323000" />
<workItem from="1629365110577" duration="847000" />
<workItem from="1629771678937" duration="537000" />
<workItem from="1629774458891" duration="812000" />
<workItem from="1629854869947" duration="503000" />
<workItem from="1630466216768" duration="33000" />
<workItem from="1630466362401" duration="562000" />
</task>
<task id="LOCAL-00001" summary="1. 新增自动检测安装ansible(本地yum源的方式实现)&#10;2. 抽取后的变量放回各自的roles中(因不想对客户暴露)&#10;3. clickhous初始化脚本关于bifang-maridb的部分变量化&#10;4. 修复一些小bug">
<created>1629774644583</created>
<option name="number" value="00001" />
<option name="presentableId" value="LOCAL-00001" />
<option name="project" value="LOCAL" />
<updated>1629774644583</updated>
</task>
<option name="localTasksCounter" value="2" />
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="1707000" />
<option name="totallyTimeSpent" value="3617000" />
</component>
<component name="ToolWindowManager">
<frame x="0" y="0" width="2560" height="1440" extended-state="0" />
<editor active="true" />
<layout>
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25377283" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.26171565" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
@@ -140,19 +320,23 @@
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value="1. 新增自动检测安装ansible(本地yum源的方式实现)&#10;2. 抽取后的变量放回各自的roles中(因不想对客户暴露)&#10;3. clickhous初始化脚本关于bifang-maridb的部分变量化&#10;4. 修复一些小bug" />
<option name="LAST_COMMIT_MESSAGE" value="1. 新增自动检测安装ansible(本地yum源的方式实现)&#10;2. 抽取后的变量放回各自的roles中(因不想对客户暴露)&#10;3. clickhous初始化脚本关于bifang-maridb的部分变量化&#10;4. 修复一些小bug" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/install.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="330">
<caret line="22" column="10" lean-forward="true" selection-start-line="22" selection-start-column="10" selection-end-line="22" selection-end-column="10" />
<state relative-caret-position="689">
<caret line="53" column="22" lean-forward="true" selection-start-line="53" selection-start-column="22" selection-end-line="53" selection-end-column="22" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="FIRST">
<first_editor relative-caret-position="960">
<caret line="64" column="27" selection-start-line="64" selection-start-column="27" selection-end-line="64" selection-end-column="27" />
<state split_layout="SECOND">
<first_editor relative-caret-position="1005">
<caret line="67" column="10" selection-start-line="67" selection-start-column="10" selection-end-line="67" selection-end-column="10" />
</first_editor>
<second_editor />
</state>

81
README.md
View File

@@ -1,81 +0,0 @@
# 使用手册
[TOC]
## 目录结构
Galaxy-auto-deploy-cluster-21.07
├── inventories
│   └── pro
│   ├── group_vars
│   └── hosts
├── parcels
│   ├── menu
│   │   ├── appMenu
│   │   ├── commonMenu
│   │   └── homePage
│   ├── roles
│   │   ├── check
│   │   ├── components
│   │   ├── dependencies
│   │   ├── init
│   │   └── services
│   ├── Arangodb.yml
│   ├── check-components.yml
│   ├── check-services.yml
│   ├── Clickhouse.yml
│   ├── Druid.yml
│   ├── environment.yml
│   ├── galaxy-chproxy.yml
│   ├── galaxy-gateway-keepalive.yml
│   ├── galaxy-gateway-nginx.yml
│   ├── galaxy-gohangout.yml
│   ├── galaxy-hos-keepalive.yml
│   ├── galaxy-hos-nginx.yml
│   ├── galaxy-hos-service.yml
│   ├── galaxy-job-service.yml
│   ├── galaxy-qgw-service.yml
│   ├── galaxy-report-service.yml
│   ├── Hadoop.yml
│   ├── HBase.yml
│   ├── init-clickhouse.yml
│   ├── init-druid.yml
│   ├── init-galaxy-gateway-keepalive.yml
│   ├── init-galaxy-hos-service.yml
│   ├── init-galaxy-job-service.yml
│   ├── init-hbase.yml
│   ├── init-kafka.yml
│   ├── init-spark.yml
│   ├── init-storm.yml
│   ├── Kafka.yml
│   ├── Mariadb.yml
│   ├── Nacos.yml
│   ├── Spark.yml
│   ├── Storm.yml
│   ├── test1.yml
│   ├── test.yml
│   └── Zookeeper.yml
├── software_packages
## 纪要
- 主机ip配置文件
inventories/pro/hosts
- 所有role(主机)使用的变量
inventories/pro/group_vars/all
`注意`: 新增变量名不可与已有变量发生重复
- 部署包目录
software_packages
为方便管理, 将相关部署包和镜像移到software_packages文件夹下(避免大文件上传gitlab)
## 使用说明
1. 配置 inventories/pro/hosts 部署组件的主机
2. 视情况修改 inventories/pro/group_vars/all 里的相关配置
3. 确认好配置后执行 "sh install.sh" 进行安装

152
inventories/pro/group_vars/all → configurations/components.yml
View File

@@ -1,110 +1,3 @@
#######################
#### ####
#### 基础配置 ####
#### ####
#######################
#============================基础配置===============================#
#根据对应的地区修改;默认为东八区，若为东六区，则修改为:"+06:00"
local_time: "+08:00"
#若使用本安装包安装jdk，则此配置无需更改；若使用其他安装包或已安装过，则指向已有的jdk安装目录
java_home: /usr/lib/jvm/jdk1.8.0_73
#统一安装路径
install_path: /home/ceiec
#数据存储路径，包含数据和日志
data_path: /home/ceiec
#若安装过程中检测到对应组件已存在，是否允许脚本卸载已存在的组件。
#仅限本安装包部署的组件。
allowed_unload: "yes"
# 所有安装包路径,可自己指定绝对路径
package_path: "{{ playbook_dir | dirname }}/software_packages"
#package_path: /opt/software_packages
#============================bifang-Mariadb===============================#
#Bifang MariaDB 地址，用于galaxy-qgw-service、galaxy-report-service的业务使用。
bifang_mariadb_host: 192.168.40.153
#Bifang mariadb 端口
bifang_mariadb_port: 3306
#Bifang mariadb 数据库名
bifang_mariadb_database: tsg-bifang
#Bifang mariadb pin
bifang_mariadb_pin: 111111
#galaxy mariadb key
galaxy_mariadb_pin: ceiec2019
#============================虚IP===============================#
#-------------apps------------------#
#Galaxy 业务组件keepalive IP地址,填写的IP只能是和hosts-services配置的IP是同段的，且不能跟同段内已有的IP相同。
gateway_keepalive_host: 192.168.45.252
#Galaxy 业务组件keepalive绑定网卡名称，不支持两个服务器网卡名称不同的情况，此情况需要安装完成后手动修改配置文件
gateway_keepalive_interface: eth0
#Galaxy 业务组件keepalive router_id 注意同网段内此值不可重复
gateway_keepalive_router_id: 61
#-------------hos------------------#
#是否需要对hos配置keepalive，若不需要则此处为no，若需要则为yes
#此项配置用于在服务器较少环境下，不对galaxy-hos-service服务单独做负载均衡的情况。
hos_keepalive_need: "yes"
#galaxy-hos-service服务keepalive IP地址 若 hos_keepalive_need为no，则此处填写gateway_keepalive_host对应的IP
#填写的IP只能是和hosts-hoskeepalive配置的IP是同段的，且不能跟同段内已有的IP相同。
hos_keepalive_host: 192.168.45.253
#galaxy-hos-service服务keepalive绑定网卡名称，不支持两个服务器网卡名称不同的情况，此情况需要安装完成后手>动修改配置文件
#若 hos_keepalive_need 配置no则此处 可不进行修改
hos_keepalive_interface: eth0
#galaxy-hos-service服务keepalive router_id 注意同网段内此值不可重复
#若 hos_keepalive_need 配置no则此处 可不进行修改
hos_keepalive_router_id: 62
#============================Kafka===============================#
#任务消费kafka地址,若无特殊情况为本安装集群kafka地址，格式:kafkaip1:9092,kafkaip2:9092,kafkaip3:9092.....
consumer_kafka_servers: 192.168.45.37:9092,192.168.45.38:9092,192.168.45.39:9092
#任务结果写入kafka地址,若无特殊情况为本安装集群kafka地址，格式:kafkaip1:9092,kafkaip2:9092,kafkaip3:9092.....
producer_kafka_servers: 192.168.45.37:9092,192.168.45.38:9092,192.168.45.39:9092
#============================默认配置===============================#
#自研服务组件使用nacos 命名空间名称
services_config_namespace: prod
#Galaxy的 Nacos pin
nacos_pin: nacos
#Nacos 使用的mariadb 数据库名称
mariadb_nacos_database: nacos
#Druid 使用的mariadb 数据库名称
mariadb_druid_database: druid
#galaxy-job-service依赖的MariaDB数据库名称
mariadb_job_database: xxl_job
########################
##### ####
##### 组件配置 ####
##### ####
########################
#===========================druid===============================#
#druid服务使用磁盘最大大小，若druid单独使用服务器则配置，服务器可用空间的 75%，单位 字节
#原配置名称：druid.server.maxSize
@@ -229,6 +122,9 @@ topic_file_bytes: 1073741824
#缓冲池字节大小 单位，单位：MB
mariadb_innodb_buffer_pool_size: 1024
#galaxy mariadb key
galaxy_mariadb_pin: galaxy2019
#===========================Spark===============================#
#spark worker 节点内存，单位：MB
spark_worker_mem: 512
@@ -250,6 +146,21 @@ storm_worker_max_mem: 1024
#单个worker内存大小，单位：MB
storm_worker_min_mem: 1024
#===========================Flink================================#
#网络缓存大小
taskmanager_memory_network_min: 512m
taskmanager_memory_network_max: 1280m
#taskmanager堆外内存
taskmanager_memory_managed_size: 256m
#TaskManager进程占用的所有与Flink相关的内存
taskmanager_memory_flink_size: 5120m
#JobManager进程占用的所有与Flink相关的内存
jobmanager_memory_flink_size: 1024m
#===========================Zookeeper===============================#
#zookeeper 进程 启动内存大小 单位MB
zookeeper_max_mem: 1024
@@ -261,30 +172,3 @@ galaxy_mariadb_port: 3306
#mariadb 用户
galaxy_mariadb_username: root
########################
##### ####
##### 业务配置 ####
##### ####
########################
#=======================galaxy-hos-service==========================#
#galaxy-hos-service服务 tomcat缓存大小,单位KB
hos_tomcat_cache_size: 100000
#galaxy-hos-service 服务内存设置
hos_java_opts: "-Xmx512m -Xms512m -Xmn512m"
#=======================galaxy-qgw-service==========================#
#galaxy-qgw-service 服务内存设置
qgw_java_opts: "-Xmx512m -Xms512m"
#=======================galaxy-job-service==========================#
#galaxy-job-service 服务内存设置
job_java_opts: "-Xmx512m -Xms512m"
#======================galaxy-report-service=========================#
#galaxy-report-service 服务内存设置
report_java_opts: "-Xmx512m -Xms512m"

91
configurations/config.yml Normal file
View File

@@ -0,0 +1,91 @@
#============================基础配置===============================#
#根据对应的地区修改;默认为东八区，若为东六区，则修改为:"+06:00"
local_time: "+08:00"
#若使用本安装包安装jdk，则此配置无需更改；若使用其他安装包或已安装过，则指向已有的jdk安装目录
java_home: /usr/lib/jvm/jdk1.8.0_73
#统一安装路径
install_path: /home/galaxy
#数据存储路径，包含数据和日志
data_path: /home/galaxy
#若安装过程中检测到对应组件已存在，是否允许脚本卸载已存在的组件。
#仅限本安装包部署的组件。
allowed_unload: "yes"
# 所有安装包路径,可自己指定绝对路径
package_path: "{{ playbook_dir | dirname }}/software_packages"
#package_path: /opt/software_packages
#============================bifang-Mariadb===============================#
#Bifang MariaDB 地址，用于galaxy-qgw-service、galaxy-report-service、flink-dos的业务使用。
bifang_mariadb_host: 192.168.40.153
#Bifang mariadb 端口
bifang_mariadb_port: 3306
#Bifang mariadb 数据库名
bifang_mariadb_database: tsg-bifang
#Bifang mariadb root's pin
bifang_mariadb_pin: 111111
#============================虚IP===============================#
#-------------apps------------------#
#Galaxy 业务组件keepalive IP地址,填写的IP只能是和hosts-services配置的IP是同段的，且不能跟同段内已有的IP相同。
gateway_keepalive_host: 192.168.45.252
#Galaxy 业务组件keepalive绑定网卡名称，不支持两个服务器网卡名称不同的情况，此情况需要安装完成后手动修改配置文件
gateway_keepalive_interface: eth0
#Galaxy 业务组件keepalive router_id 注意同网段内此值不可重复
gateway_keepalive_router_id: 61
#-------------hos------------------#
#是否需要对hos配置keepalive，若不需要则此处为no，若需要则为yes
#此项配置用于在服务器较少环境下，不对galaxy-hos-service服务单独做负载均衡的情况。
hos_keepalive_need: "yes"
#galaxy-hos-service服务keepalive IP地址 若 hos_keepalive_need为no，则此处填写gateway_keepalive_host对应的IP
#填写的IP只能是和hosts-hoskeepalive配置的IP是同段的，且不能跟同段内已有的IP相同。
hos_keepalive_host: 192.168.45.253
#galaxy-hos-service服务keepalive绑定网卡名称，不支持两个服务器网卡名称不同的情况，此情况需要安装完成后手>动修改配置文件
#若 hos_keepalive_need 配置no则此处 可不进行修改
hos_keepalive_interface: eth0
#galaxy-hos-service服务keepalive router_id 注意同网段内此值不可重复
#若 hos_keepalive_need 配置no则此处 可不进行修改
hos_keepalive_router_id: 62
#============================clickhouse===============================#
#ssd磁盘挂载的目录, 如果没有ssd磁盘可选择一块普通磁盘
clickhouse_ssd_path: /ssd_data/clickhouse/
#============================Kafka===============================#
#任务消费kafka地址,若无特殊情况为本安装集群kafka地址，格式:kafkaip1:9092,kafkaip2:9092,kafkaip3:9092.....
kafka_source_servers: 192.168.45.37:9092,192.168.45.38:9092,192.168.45.39:9092
#任务结果写入kafka地址,若无特殊情况为本安装集群kafka地址，格式:kafkaip1:9092,kafkaip2:9092,kafkaip3:9092.....
kafka_sink_servers: 192.168.45.37:9092,192.168.45.38:9092,192.168.45.39:9092
#============================默认配置===============================#
#自研服务组件使用nacos 命名空间名称
services_config_namespace: prod
#Galaxy的 Nacos pin
nacos_pin: nacos
#Nacos 使用的mariadb 数据库名称
mariadb_nacos_database: nacos
#Druid 使用的mariadb 数据库名称
mariadb_druid_database: druid
#galaxy-job-service依赖的MariaDB数据库名称
mariadb_job_database: xxl_job

13
inventories/pro/hosts → configurations/hosts
View File

@@ -45,7 +45,13 @@
192.168.45.39
#此标签指定的IP用于：Apache Storm的安装，根据内部约定，若集群整体小于3台则第一台为numbus节点；大于三台则前三台为nimbus节点。
[storm]
#[storm]
#192.168.45.37
#192.168.45.38
#192.168.45.39
#此标签指定的IP用于：Apache Flink的安装，根据内部约定，前两台台为master节点；所有节点为worker节点。
[flink]
192.168.45.37
192.168.45.38
192.168.45.39
@@ -84,3 +90,8 @@
192.168.45.41
192.168.45.42
192.168.45.43
#dos检测的离线generate-baseline程序
[dos_baseline]
192.168.45.43

18
configurations/services.yml Normal file
View File

@@ -0,0 +1,18 @@
#=======================galaxy-hos-service==========================#
#galaxy-hos-service服务 tomcat缓存大小,单位KB
hos_tomcat_cache_size: 100000
#galaxy-hos-service 服务内存设置
hos_java_opts: "-Xmx512m -Xms512m -Xmn512m"
#=======================galaxy-qgw-service==========================#
#galaxy-qgw-service 服务内存设置
qgw_java_opts: "-Xmx512m -Xms512m"
#=======================galaxy-job-service==========================#
#galaxy-job-service 服务内存设置
job_java_opts: "-Xmx512m -Xms512m"
#======================galaxy-report-service=========================#
#galaxy-report-service 服务内存设置
report_java_opts: "-Xmx512m -Xms512m"

47
install.sh
View File

@@ -60,7 +60,7 @@ if [[ $hasAnsible != '0' ]]; then
else
mv /etc/yum.repos.d/bak_`date '+%Y-%m-%d'`/* /etc/yum.repos.d/
rm -r /etc/yum.repos.d/bak_`date '+%Y-%m-%d'`
echo -e "\033[32;1mThis Ansible not installed, please check it manually\033[0m"
echo -e "\033[32;1mThis Ansible install failed, please check it manually\033[0m"
exit 1
fi
else
@@ -79,7 +79,7 @@ function checkCluster() {
clear
echo "Check the environment to be deployed, including JDK, PIP, Docker, Firewall, TimeZone if not installed"
sleep 2
ansible-playbook -i ../inventories/pro/hosts environment.yml
ansible-playbook -i ../configurations/hosts environment.yml
if [ $? -eq '0' ];then
clear
echo -e "\033[32;1mEnvironmental check complete, ready for installation\033[0m"
@@ -150,7 +150,7 @@ NAME=$1
is=`getrecordnum $NAME`
if [ $is -lt '1' ];then
ansible-playbook -i ../inventories/pro/hosts $NAME.yml
ansible-playbook -i ../configurations/hosts $NAME.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $NAME
@@ -165,7 +165,7 @@ else
read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn
case $yn in
[Yy]*)
ansible-playbook -i ../inventories/pro/hosts $NAME.yml
ansible-playbook -i ../configurations/hosts $NAME.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $NAME
@@ -193,13 +193,15 @@ function installAllCommon() {
clear
i=0
ins_names=(Zookeeper Mariadb galaxy-gateway-keepalive Nacos Kafka Storm Hadoop HBase Clickhouse Druid Spark Arangodb)
#ins_names=(Zookeeper Mariadb galaxy-gateway-keepalive Nacos Kafka Storm Hadoop HBase Clickhouse Druid Spark Arangodb)
#替换Storm成Flink
ins_names=(Zookeeper Mariadb galaxy-gateway-keepalive Nacos Kafka Flink Hadoop HBase Clickhouse Druid Spark Arangodb)
echo ${ins_names[@]} > $NUM_DIR/common_list
#根据组合进行顺序安装
for element in ${ins_names[@]}
do
ansible-playbook -i ../inventories/pro/hosts $element.yml
ansible-playbook -i ../configurations/hosts $element.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $element
@@ -232,7 +234,7 @@ read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn
oldnums=${#ins_names[*]}
for element in ${ins_names[@]}
do
ansible-playbook -i ../inventories/pro/hosts $element.yml
ansible-playbook -i ../configurations/hosts $element.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $element
@@ -267,7 +269,9 @@ done
function installcombination() {
if [ -f $NUM_DIR/common_list ];then
if [ `cat $NUM_DIR/common_list | egrep "Mariadb|Nacos|Zookeeper|galaxy-gateway-keepalive|Kafka|Storm|Hadoop|HBase|Clickhouse|Druid|Spark|Arangodb" | wc -l` -gt '0' ]
#if [ `cat $NUM_DIR/common_list | egrep "Mariadb|Nacos|Zookeeper|galaxy-gateway-keepalive|Kafka|Storm|Hadoop|HBase|Clickhouse|Druid|Spark|Arangodb" | wc -l` -gt '0' ]
#替换Storm成Flink
if [ `cat $NUM_DIR/common_list | egrep "Mariadb|Nacos|Zookeeper|galaxy-gateway-keepalive|Kafka|Flink|Hadoop|HBase|Clickhouse|Druid|Spark|Arangodb" | wc -l` -gt '0' ]
then
abnormalCombination
else
@@ -289,7 +293,7 @@ NAME=$1
is=`getrecordnum $NAME`
if [ $is -lt '1' ];then
ansible-playbook -i ../inventories/pro/hosts $NAME.yml
ansible-playbook -i ../configurations/hosts $NAME.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $NAME
@@ -304,7 +308,7 @@ else
read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn
case $yn in
[Yy]*)
ansible-playbook -i ../inventories/pro/hosts $NAME.yml
ansible-playbook -i ../configurations/hosts $NAME.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $NAME
@@ -332,7 +336,7 @@ function installAllApps() {
clear
i=0
hoskeep=`cat ../inventories/pro/group_vars/all | grep -vE "^#|^$" | grep "hos_keepalive_need" | grep yes | wc -l`
hoskeep=`cat ../configurations/config.yml | grep -vE "^#|^$" | grep "hos_keepalive_need" | grep yes | wc -l`
if [[ $hoskeep -eq "1" ]]; then
ins_names=(galaxy-gateway-nginx galaxy-qgw-service galaxy-job-service galaxy-report-service galaxy-hos-service galaxy-chproxy galaxy-hos-nginx galaxy-hos-keepalive galaxy-gohangout)
else
@@ -343,7 +347,7 @@ echo ${ins_names[@]} > $NUM_DIR/app_list
#根据组合进行顺序安装
for element in ${ins_names[@]}
do
ansible-playbook -i ../inventories/pro/hosts $element.yml
ansible-playbook -i ../configurations/hosts $element.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $element
@@ -376,7 +380,7 @@ read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn
oldnums=${#ins_names[*]}
for element in ${ins_names[@]}
do
ansible-playbook -i ../inventories/pro/hosts $element.yml
ansible-playbook -i ../configurations/hosts $element.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $element
@@ -430,13 +434,15 @@ function initAll() {
clear
i=0
ins_names=(init-galaxy-gateway-keepalive init-kafka init-hbase init-druid init-clickhouse init-storm init-spark init-galaxy-hos-service)
#ins_names=(init-galaxy-gateway-keepalive init-kafka init-hbase init-druid init-clickhouse init-storm init-spark init-galaxy-hos-service init-dos)
#替换init-storm成init-flink
ins_names=(init-galaxy-gateway-keepalive init-kafka init-hbase init-druid init-clickhouse init-flink init-spark init-galaxy-hos-service init-dos)
echo ${ins_names[@]} > $NUM_DIR/init_list
#根据组合进行顺序安装
for element in ${ins_names[@]}
do
ansible-playbook -i ../inventories/pro/hosts $element.yml
ansible-playbook -i ../configurations/hosts $element.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $element
@@ -470,7 +476,7 @@ read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn
oldnums=${#ins_names[*]}
for element in ${ins_names[@]}
do
ansible-playbook -i ../inventories/pro/hosts $element.yml
ansible-playbook -i ../configurations/hosts $element.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $element
@@ -503,7 +509,10 @@ done
function installInit() {
if [ -f $NUM_DIR/init_list ];then
if [ `cat $NUM_DIR/init_list | egrep "init-galaxy-gateway-keepalive|init-kafka|init-hbase|init-druid|init-clickhouse|init-storm|init-spark|init-galaxy-job-service|init-galaxy-hos-service" | wc -l` -gt '0' ]
#if [ `cat $NUM_DIR/init_list | egrep "init-galaxy-gateway-keepalive|init-kafka|init-hbase|init-druid|init-clickhouse|init-storm|init-spark|init-galaxy-job-service|init-galaxy-hos-service|init-dos" | wc -l` -gt '0' ]
#替换init-storm成init-flink
if [ `cat $NUM_DIR/init_list | egrep "init-galaxy-gateway-keepalive|init-kafka|init-hbase|init-druid|init-clickhouse|init-flink|init-spark|init-galaxy-job-service|init-galaxy-hos-service|init-dos" | wc -l` -gt '0' ]
then
abnormalInit
else
@@ -529,7 +538,7 @@ echo ${ins_names[@]} > $NUM_DIR/check_list
#根据组合进行顺序安装
for element in ${ins_names[@]}
do
ansible-playbook -i ../inventories/pro/hosts $element.yml
ansible-playbook -i ../configurations/hosts $element.yml
if [ $? -eq '0' ];then
echoSuccess $element
unset ins_names[$i]
@@ -566,7 +575,7 @@ read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn
oldnums=${#ins_names[*]}
for element in ${ins_names[@]}
do
ansible-playbook -i ../inventories/pro/hosts $element.yml
ansible-playbook -i ../configurations/hosts $element.yml
if [ $? -eq '0' ];then
sleep 10
echoSuccess $element

8
parcels/Arangodb.yml
View File

@@ -2,7 +2,7 @@
remote_user: root
roles:
- components/arangodb
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - roles/components/arangodb/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- roles/components/arangodb/vars/main.yml

24
parcels/Clickhouse.yml
View File

@@ -3,26 +3,26 @@
remote_user: root
roles:
- components/clickhouse/common
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - roles/components/clickhouse/common/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- roles/components/clickhouse/common/vars/main.yml
- hosts: clickhouse
remote_user: root
roles:
- components/clickhouse/query
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - roles/components/clickhouse/common/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- roles/components/clickhouse/common/vars/main.yml
- hosts: clickhouse
remote_user: root
roles:
- components/clickhouse/data
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - roles/components/clickhouse/common/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- roles/components/clickhouse/common/vars/main.yml

8
parcels/Druid.yml
View File

@@ -2,7 +2,7 @@
remote_user: root
roles:
- components/druid
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - roles/components/druid/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- roles/components/druid/vars/main.yml

11
parcels/Flink.yml Normal file
View File

@@ -0,0 +1,11 @@
- hosts: flink
remote_user: root
roles:
- components/flink
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/flink/vars/main.yml

10
parcels/HBase.yml
View File

@@ -2,9 +2,9 @@
remote_user: root
roles:
- components/hbase
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/hbase/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/hbase/vars/main.yml

10
parcels/Hadoop.yml
View File

@@ -2,10 +2,10 @@
remote_user: root
roles:
- components/hadoop
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/hadoop/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/hadoop/vars/main.yml

10
parcels/Kafka.yml
View File

@@ -3,9 +3,9 @@
remote_user: root
roles:
- components/kafka
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/kafka/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/kafka/vars/main.yml

20
parcels/Mariadb.yml
View File

@@ -3,18 +3,18 @@
remote_user: root
roles:
- components/mariadb/install-mariadb-cluster
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/mariadb/install-mariadb-cluster/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/mariadb/install-mariadb-cluster/vars/main.yml
- hosts: mariadb
remote_user: root
roles:
- components/mariadb/mariadb-double-master
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/mariadb/install-mariadb-cluster/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/mariadb/install-mariadb-cluster/vars/main.yml

10
parcels/Nacos.yml
View File

@@ -3,8 +3,8 @@
remote_user: root
roles:
- components/nacos
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/nacos/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/nacos/vars/main.yml

10
parcels/Spark.yml
View File

@@ -3,8 +3,8 @@
remote_user: root
roles:
- components/spark
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/spark/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/spark/vars/main.yml

10
parcels/Storm.yml
View File

@@ -3,9 +3,9 @@
remote_user: root
roles:
- components/storm
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/storm/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/storm/vars/main.yml

10
parcels/Zookeeper.yml
View File

@@ -4,9 +4,9 @@
# max_fail_percentage: 0
roles:
- components/zookeeper
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/zookeeper/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/zookeeper/vars/main.yml

4
parcels/check-components.yml
View File

@@ -2,5 +2,5 @@
remote_user: root
roles:
- check/check-components
# vars_files:
# - ../configurations/config.yml
vars_files:
- ../configurations/config.yml

4
parcels/check-services.yml
View File

@@ -2,5 +2,5 @@
remote_user: root
roles:
- check/check-services
# vars_files:
# - ../configurations/config.yml
vars_files:
- ../configurations/config.yml

16
parcels/environment.yml
View File

@@ -3,9 +3,9 @@
remote_user: root
roles:
- dependencies/install-jdk
# vars_files:
# - roles/dependencies/install-jdk/vars/main.yml
# - ../configurations/config.yml
vars_files:
- roles/dependencies/install-jdk/vars/main.yml
- ../configurations/config.yml
#安装pip和docker
- hosts: all
@@ -13,13 +13,13 @@
roles:
- dependencies/install-docker
- dependencies/install-pip
# vars_files:
# - ../configurations/config.yml
vars_files:
- ../configurations/config.yml
- hosts: all
remote_user: root
roles:
- dependencies/check-env
# vars_files:
# - roles/dependencies/check-env/vars/main.yml
# - ../configurations/config.yml
vars_files:
- roles/dependencies/check-env/vars/main.yml
- ../configurations/config.yml

10
parcels/galaxy-chproxy.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- services/galaxy-chproxy
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-chproxy/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-chproxy/vars/main.yml

10
parcels/galaxy-gateway-keepalive.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- services/galaxy-gateway-keepalive
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-gateway-keepalive/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-gateway-keepalive/vars/main.yml

10
parcels/galaxy-gateway-nginx.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- services/galaxy-gateway-nginx
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-gateway-nginx/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-gateway-nginx/vars/main.yml

10
parcels/galaxy-gohangout.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- services/galaxy-gohangout
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-gohangout/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-gohangout/vars/main.yml

10
parcels/galaxy-hos-keepalive.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- services/galaxy-hos-keepalive
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-hos-keepalive/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-hos-keepalive/vars/main.yml

10
parcels/galaxy-hos-nginx.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- services/galaxy-hos-nginx
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-hos-nginx/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-hos-nginx/vars/main.yml

10
parcels/galaxy-hos-service.yml
View File

@@ -2,9 +2,9 @@
remote_user: root
roles:
- services/galaxy-hos-service
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-hos-service/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-hos-service/vars/main.yml

10
parcels/galaxy-job-service.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- services/galaxy-job-service
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-job-service/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-job-service/vars/main.yml

10
parcels/galaxy-qgw-service.yml
View File

@@ -2,9 +2,9 @@
remote_user: root
roles:
- services/galaxy-qgw-service
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-qgw-service/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-qgw-service/vars/main.yml

10
parcels/galaxy-report-service.yml
View File

@@ -2,9 +2,9 @@
remote_user: root
roles:
- services/galaxy-report-service
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-report-service/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-report-service/vars/main.yml

10
parcels/init-clickhouse.yml
View File

@@ -3,8 +3,8 @@
remote_user: root
roles:
- init/clickhouse
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/clickhouse/common/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/clickhouse/common/vars/main.yml

9
parcels/init-dos.yml Normal file
View File

@@ -0,0 +1,9 @@
- hosts:
- dos_baseline
remote_user: root
roles:
- init/dos-baseline
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml

10
parcels/init-druid.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- init/druid
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/components/druid/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/components/druid/vars/main.yml

10
parcels/init-flink.yml Normal file
View File

@@ -0,0 +1,10 @@
- hosts:
- flink
remote_user: root
roles:
- init/flink
vars_files:
- roles/components/flink/vars/main.yml
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml

13
parcels/init-galaxy-gateway-keepalive.yml
View File

@@ -2,8 +2,11 @@
remote_user: root
roles:
- init/galaxy-gateway-keepalive
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-gateway-keepalive/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-gateway-keepalive/vars/main.yml
- roles/services/galaxy-qgw-service/vars/main.yml
- roles/services/galaxy-chproxy/vars/main.yml
- roles/services/galaxy-gateway-nginx/vars/main.yml

10
parcels/init-galaxy-hos-service.yml
View File

@@ -2,8 +2,8 @@
remote_user: root
roles:
- init/galaxy-hos-service
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-hos-service/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-hos-service/vars/main.yml

10
parcels/init-galaxy-job-service.yml
View File

@@ -3,8 +3,8 @@
remote_user: root
roles:
- init/galaxy-job-service
# vars_files:
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
# - roles/services/galaxy-job-service/vars/main.yml
vars_files:
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml
- roles/services/galaxy-job-service/vars/main.yml

10
parcels/init-hbase.yml
View File

@@ -3,8 +3,8 @@
remote_user: root
roles:
- init/hbase
# vars_files:
# - roles/components/hbase/vars/main.yml
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
vars_files:
- roles/components/hbase/vars/main.yml
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml

10
parcels/init-kafka.yml
View File

@@ -3,8 +3,8 @@
remote_user: root
roles:
- init/kafka
# vars_files:
# - roles/components/kafka/vars/main.yml
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
vars_files:
- roles/components/kafka/vars/main.yml
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml

10
parcels/init-spark.yml
View File

@@ -3,8 +3,8 @@
remote_user: root
roles:
- init/spark
# vars_files:
# - roles/components/spark/vars/main.yml
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
vars_files:
- roles/components/spark/vars/main.yml
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml

10
parcels/init-storm.yml
View File

@@ -3,8 +3,8 @@
remote_user: root
roles:
- init/storm
# vars_files:
# - roles/components/storm/vars/main.yml
# - ../configurations/config.yml
# - ../configurations/components.yml
# - ../configurations/services.yml
vars_files:
- roles/components/storm/vars/main.yml
- ../configurations/config.yml
- ../configurations/components.yml
- ../configurations/services.yml

2
parcels/menu/commonMenu
View File

@@ -5,7 +5,7 @@ Open Source Software：
├     [*] Mariadb         ┤
├     [*] Nacos ┤
├     [*] Apache Kafka ┤
├ [*] Apache Storm ┤
├ [*] Apache Flink ┤
├ [*] Apache Hadoop ┤
├ [*] Apache HBase ┤
├ [*] Clickhouse ┤

2
parcels/menu/homePage
View File

@@ -1,4 +1,4 @@
Galaxy Data Platform Cluster - TSG21.06
Galaxy Data Platform Cluster - TSG21.08
Please choose one of the following:

23
parcels/roles/check/check-components/tasks/main.yml
View File

@@ -1,15 +1,22 @@
- name: Checking Strom Topology Num
shell: "source /etc/profile && storm list | grep 'ACTIVE' | wc -l"
register: stormlist
#- name: Checking Strom Topology Num
# shell: "source /etc/profile && storm list | grep 'ACTIVE' | wc -l"
# register: stormlist
# run_once: true
# delegate_to: '{{ groups.storm[0] }}'
- name: Checking Flink Topology Num
shell: "source /etc/profile && flink list | grep 'RUNNING' | wc -l"
register: flinklist
run_once: true
delegate_to: '{{ groups.storm[0] }}'
delegate_to: '{{ groups.flink[0] }}'
- name: Checking Strom Topology Num
fail:
msg: "storm 提交任务数量与版本对应任务数量不匹配，请检查"
when: stormlist.stdout < '15'
msg: "flink 提交任务数量与版本对应任务数量不匹配，请检查"
when: flinklist.stdout < '16'
run_once: true
delegate_to: '{{ groups.storm[0] }}'
delegate_to: '{{ groups.flink[0] }}'
- name: Checking Gohangout Num
shell: "ps -ef | grep gohangout | grep worker | grep -v grep | wc -l"
@@ -20,6 +27,6 @@
- name: Checking Gohangout Num
fail:
msg: "Gohangout 任务启动数据量与版本对应任务数量不匹配，请检查"
when: golist.stdout < '10'
when: golist.stdout < '12'
run_once: true
delegate_to: '{{ groups.gohangout[0] }}'

2
parcels/roles/check/check-services/tasks/main.yml
View File

@@ -60,7 +60,7 @@
- fail:
msg: "{{ inventory_hostname }}- gohangout start job num is not all. Please check it"
when: check_out.stdout < '10'
when: check_out.stdout < '12'
run_once: true
delegate_facts: true
delegate_to: '{{ groups.gohangout[1] }}'

3
parcels/roles/components/arangodb/files/README.md Normal file
View File

@@ -0,0 +1,3 @@
#权限
- 账户: query
- 密码: galaxy2018

BIN
parcels/roles/components/arangodb/files/arangodb.tar.gz
View File

Binary file not shown.

BIN
parcels/roles/components/arangodb/files/conf.bak.tar
View File

Binary file not shown.

2
parcels/roles/components/arangodb/vars/main.yml
View File

@@ -1,5 +1,5 @@
#注意这里面定义的key中不能包含减号-,否则程序无法识别定义的key
arangodb_password: ceiec2018
arangodb_password: galaxy2018
#离线模式下使用:bifang-api的docker镜像文件名称,在../file中的文件名称
arangodb_image_tar_name: arangodb-3.6.4.tar

12
parcels/roles/components/clickhouse/common/tasks/install_clickhouse.yml
View File

@@ -1,3 +1,8 @@
- name: create {{ install_path }}
file:
state: directory
path: '{{ install_path }}'
- name: Check whether the CPU supports the SSE4_2 instruction set
shell: lscpu |grep sse4_2 | wc -l
register: check_cpu_out
@@ -17,7 +22,7 @@
force: true
mode: 0755
when: check_out.stdout >= '1'
- name: unload kafka
- name: unload clickhouse
shell: cd {{ install_path }} && sh unload_ck.sh
when: check_out.stdout >= '1'
- name: Ansible delete {{ install_path }}/unload_ck.sh
@@ -59,6 +64,11 @@
run_once: true
delegate_to: 127.0.0.1
- name: create clickhouse ssd data path:{{ clickhouse_ssd_path }}
file:
state: directory
path: '{{ clickhouse_ssd_path }}'
- name: create clickhouse package path:{{ install_path }}/clickhouse
file:
state: directory

22
parcels/roles/components/clickhouse/common/templates/config.xml.j2
View File

@@ -368,6 +368,28 @@
-->
<format_schema_path>{{ install_path }}/clickhouse/format_schemas/</format_schema_path>
<storage_configuration>
<disks>
<ssd>
<path>{{ clickhouse_ssd_path }}</path>
</ssd>
</disks>
<policies>
<ssd_to_hdd>
<volumes>
<hot>
<disk>ssd</disk>
</hot>
<default>
<disk>default</disk>
</default>
</volumes>
<move_factor>0.1</move_factor>
</ssd_to_hdd>
</policies>
</storage_configuration>
<!-- Uncomment to disable ClickHouse internal DNS caching. -->
<!-- <disable_internal_dns_cache>1</disable_internal_dns_cache> -->
</yandex>

2
parcels/roles/components/clickhouse/common/templates/start_exporter.sh.j2
View File

@@ -6,4 +6,4 @@ docker rm -f clickhouse_exporter
sleep 3
docker run -d -p 9904:9116 --name clickhouse_exporter --restart always --env CLICKHOUSE_USER=default --env CLICKHOUSE_PASSWORD=ceiec2019 clickhouse-exporter:v1.0 -scrape_uri=http://{{ inventory_hostname }}:8123/
docker run -d -p 9904:9116 --name clickhouse_exporter --restart always --env CLICKHOUSE_USER=default --env CLICKHOUSE_PASSWORD=galaxy2019 clickhouse-exporter:v1.0 -scrape_uri=http://{{ inventory_hostname }}:8123/

4
parcels/roles/components/clickhouse/common/vars/main.yml
View File

@@ -5,7 +5,7 @@ clickhouse_version: 20.3.12.112
default_database: default
#clickhouse 只读key
readonly_key: ceiec2018
readonly_key: galaxy2018
#clickhouse 管理key
root_key: ceiec2019
root_key: galaxy2019

2
parcels/roles/components/clickhouse/data/tasks/install_clickhouse.yml
View File

@@ -38,7 +38,7 @@
when: odbc_install.stdout != '0' and inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1])
- name: set dir user
shell: chown clickhouse:clickhouse -R {{ data_path }}/clickhouse/
shell: chown clickhouse:clickhouse -R {{ data_path }}/clickhouse/ && chown clickhouse:clickhouse -R {{ clickhouse_ssd_path }}
when: inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1])
- name: copy clickhouse-server to /etc/init.d/

4
parcels/roles/components/clickhouse/data/vars/main.yml
View File

@@ -5,7 +5,7 @@ clickhouse_version: 20.3.12.112
default_database: default
#clickhouse 只读key
readonly_key: ceiec2018
readonly_key: galaxy2018
#clickhouse 管理key
root_key: ceiec2019
root_key: galaxy2019

4
parcels/roles/components/clickhouse/query/vars/main.yml
View File

@@ -5,7 +5,7 @@ clickhouse_version: 20.3.12.112
default_database: default
#clickhouse 只读key
readonly_key: ceiec2018
readonly_key: galaxy2018
#clickhouse 管理key
root_key: ceiec2019
root_key: galaxy2019

2
parcels/roles/components/druid/tasks/install_druid.yml
View File

@@ -38,7 +38,7 @@
force: true
mode: 0755
when: check_out.stdout >= '1'
- name: unload kafka
- name: unload Druid
shell: cd {{ install_path }} && sh unload_druid.sh
when: check_out.stdout >= '1'
- name: check if {{ mariadb_druid_database }} exist

9
parcels/roles/components/druid/templates/keepdruidall.j2
View File

@@ -20,14 +20,15 @@ fi
;;
stop)
keeppid=`ps -ef | grep dae-druid-all.sh | grep -v grep | awk '{print $2}'`
echo "守护进程PID：$keeppid"
kill -9 $keeppid
ps -ef | grep dae-druid-all.sh | grep -v grep | awk '{print $2}' | xargs kill -9 2>/dev/null
#keeppid=`ps -ef | grep dae-druid-all.sh | grep -v grep | awk '{print $2}' | xargs kill -9 2>/dev/null`
#echo "守护进程PID：$keeppid"
#kill -9 $keeppid
echo "关闭所有druid进程"
#ps aux | grep druid | grep -v grep | awk '{print $2}' | xargs kill -9
livenum=`ps -ef |grep "druid" | grep -v grep | grep -v json | wc -l`
if [ $livenum -ne 0 ];then
ps aux|grep "druid" |grep -v grep | awk '{print $2}'| xargs kill -9
ps aux|grep "druid" |grep -v grep | awk '{print $2}'| xargs kill -9 2>/dev/null
fi
;;

127
parcels/roles/components/flink/tasks/install_flink.yml Normal file
View File

@@ -0,0 +1,127 @@
- name: partion_nums to ansible variable
set_fact: partion_nums="{{groups.flink|length}}"
- name: judge the machine number of flink
fail:
msg: "flink 集群最少需要配置3个节点，请修改configurations/hosts内对应IP列表。"
when: partion_nums < '3'
- name: check jdk 1.8_73
shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l
ignore_errors: false
register: jdk_out
- name: To terminate execution
fail:
msg: "检测到目标集群内存在JDK未安装节点，请检查"
when: jdk_out.stdout != '2'
run_once: true
delegate_to: 127.0.0.1
- name: create flink package path:{{ install_path }}
file:
state: directory
path: '{{ install_path }}'
- block:
- name: Check if the flink service already exists
shell: source /etc/profile && jps -l | grep "org.apache.flink.runtime" | grep -v grep |awk '{print $1}' | wc -l
register: check_out
- name: copy unload_flink.sh to {{ install_path }}/
template:
src: 'unload_flink.sh.j2'
dest: '{{ install_path }}/unload_flink.sh'
force: true
mode: 0755
when: check_out.stdout >= '1'
- name: unload flink
shell: cd {{ install_path }} && sh unload_flink.sh
when: check_out.stdout >= '1'
- name: Ansible delete {{ install_path }}/unload_flink.sh
file:
path: "{{ install_path }}/unload_flink.sh"
state: absent
- name: Check if the flink service already exists
shell: source /etc/profile && jps -l | grep "org.apache.flink.runtime" | grep -v grep |awk '{print $1}' | wc -l
register: check_out
- name: To terminate execution
fail:
msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装"
run_once: true
delegate_to: 127.0.0.1
when: check_out.stdout >= '1'
when: (allowed_unload) == "yes"
- block:
- name: Check if flink service already exists
shell: source /etc/profile && jps -l | grep "org.apache.flink.runtime" | grep -v grep |awk '{print $1}' | wc -l
register: check_out
- name: To terminate execution
fail:
msg: "检测到目标集群内已有flink请检查IP是否正确或停止/卸载已启动的组件"
run_once: true
delegate_to: 127.0.0.1
when: check_out.stdout >= '1'
when: (allowed_unload) == "no"
- name: get zookeeper_servers to ansible variable
set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181"
when: '(groups.zookeeper|length) == 3'
- name: get zookeeper_servers to ansible variable
set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181"
when: '(groups.zookeeper|length) >= 5'
#解压tar
- name: unpack {{ flink_version }}.tgz
unarchive:
src: '{{ package_path }}/{{ flink_version }}.tgz'
dest: '{{ install_path }}/'
- name: Config flink configuration
template:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
mode: 0644
with_items:
- { src: 'flink-conf.yaml.j2', dest: '{{ install_path }}/{{ flink_version }}/conf/flink-conf.yaml' }
- { src: 'masters.j2', dest: '{{ install_path }}/{{ flink_version }}/conf/masters' }
- { src: 'workers.j2', dest: '{{ install_path }}/{{ flink_version }}/conf/workers' }
- name: Config masters
lineinfile:
dest: "{{ install_path }}/{{ flink_version }}/conf/masters"
line: "{{ item }}:{{ flink_web_port }}"
with_items:
- '{{groups.flink[0]}}'
- '{{groups.flink[1]}}'
- name: Config workers
lineinfile:
dest: "{{ install_path }}/{{ flink_version }}/conf/workers"
line: "{{ item }}"
loop: "{{ groups.flink|list }}"
- name: Config flink scripts
template:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
mode: 0755
with_items:
- { src: 'set_flink_env.sh.j2', dest: '{{ install_path }}/{{ flink_version }}/bin/set_flink_env.sh' }
- { src: 'dae-flink.sh.j2', dest: '{{ install_path }}/{{ flink_version }}/bin/dae-flink.sh' }
- { src: 'keepflinkalive.j2', dest: '/etc/init.d/keepflinkalive' }
#配置环境变量
- name: set flink env and chkconfig keepflinkalive
shell: cd {{ install_path }}/{{ flink_version }}/bin/ && ./set_flink_env.sh
#启动flink守护
- name: start keepflinkalive service
shell: source /etc/profile && sh /etc/init.d/keepflinkalive start
when: hostvars[inventory_hostname]['ansible_env'].SSH_CONNECTION.split(' ')[2] == groups.flink[0]

1
parcels/roles/components/flink/tasks/main.yml Normal file
View File

@@ -0,0 +1 @@
- include: install_flink.yml

39
parcels/roles/components/flink/templates/dae-flink.sh.j2 Executable file
View File

@@ -0,0 +1,39 @@
#!/bin/sh
#修改BASE_DIR为安装的路径
#启动命令端口可自行指定
#JMX_PORT=9191 nohup $BASE_DIR/bin/kafka-server-start.sh $BASE_DIR/config/server.properties > /dev/null 2>&1 &
PRO_NAME=flink
BASE_DIR={{ install_path }}
VERSION={{ flink_version }}
source /etc/profile
if [ ! -f "$BASE_DIR/$VERSION/log/restart_sum.log" ]
then
touch $BASE_DIR/$VERSION/log/restart_sum.log
echo '0' > $BASE_DIR/$VERSION/log/restart_sum.log
fi
function start_flink(){
while true ; do
NUM=`ps aux | grep -w ${PRO_NAME} | grep -vE "grep|dae-flink" |wc -l`
if [ "${NUM}" -lt "1" ];then
cd $BASE_DIR/$VERSION/
bin/start-cluster.sh
OLD_NUM=`cat $BASE_DIR/$VERSION/log/restart_sum.log`
RESTART_NUM=`expr $OLD_NUM + 1`
echo $RESTART_NUM > $BASE_DIR/$VERSION/log/restart_sum.log
echo "`date "+%Y-%m-%d %H:%M:%S"` - flink服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/log/restart.log
fi
sleep 60s
done
}
start_flink

306
parcels/roles/components/flink/templates/flink-conf.yaml.j2 Normal file
View File

@@ -0,0 +1,306 @@
censed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
#==============================================================================
# Common
#==============================================================================
# The external address of the host on which the JobManager runs and can be
# reached by the TaskManagers and any clients which want to connect. This setting
# is only used in Standalone mode and may be overwritten on the JobManager side
# by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.
# In high availability mode, if you use the bin/start-cluster.sh script and setup
# the conf/masters file, this will be taken care of automatically. Yarn/Mesos
# automatically configure the host name based on the hostname of the node where the
# JobManager runs.
jobmanager.rpc.address: {{groups.zookeeper[0]}},{{groups.zookeeper[1]}}
# The RPC port where the JobManager is reachable.
env.java.opts: "-XX:+UseG1GC -XX:NewRatio=2 -XX:MaxGCPauseMillis=300
-XX:InitiatingHeapOccupancyPercent=35 -Xloggc:{{ install_path }}/{{ flink_version }}/log/gc.log
-XX:+PrintGCDetails
-XX:-OmitStackTraceInFastThrow
-XX:+PrintGCTimeStamps
-XX:+PrintGCDateStamps
-XX:+UseGCLogFileRotation
-XX:NumberOfGCLogFiles=20
-XX:GCLogFileSize=20M"
jobmanager.rpc.port: 6123
cluster.evenly-spread-out-slots: true
# The total process memory size for the JobManager.
#
# Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.
#jobmanager.memory.process.size: 1600m
#taskmanager.heap.mb: 2g
#jobmanager.heap.mb: 1g
#taskmanager使用的堆外内存的大小
taskmanager.memory.managed.size: {{ taskmanager_memory_managed_size }}
#用于TaskManager之间（shuffle、广播等）及与外部组件的数据传输
taskmanager.memory.network.min: {{ taskmanager_memory_network_min }}
taskmanager.memory.network.max: {{ taskmanager_memory_network_max }}
taskmanager.network.memory.floating-buffers-per-gate: 8
taskmanager.network.memory.buffers-per-channel: 2
#TaskManager进程占用的所有与Flink相关的内存
taskmanager.memory.flink.size: {{ taskmanager_memory_flink_size }}
#JobManager进程占用的所有与Flink相关的内存
jobmanager.memory.flink.size: {{ jobmanager_memory_flink_size }}
# The total process memory size for the TaskManager.
#
# Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.
#taskmanager.memory.process.size: 1728m
# To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.
# It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.
#
# taskmanager.memory.flink.size: 1280m
# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
taskmanager.numberOfTaskSlots: 48
# The parallelism used for programs that did not specify and other parallelism.
parallelism.default: 1
# The default file system scheme and authority.
#
# By default file paths without scheme are interpreted relative to the local
# root file system 'file:///'. Use this to override the default and interpret
# relative paths relative to a different file system,
# for example 'hdfs://mynamenode:12345'
#
# fs.default-scheme
#==============================================================================
# High Availability
#==============================================================================
# The high-availability mode. Possible options are 'NONE' or 'zookeeper'.
#
# high-availability: zookeeper
# The path where metadata for master recovery is persisted. While ZooKeeper stores
# the small ground truth for checkpoint and leader election, this location stores
# the larger objects, like persisted dataflow graphs.
#
# Must be a durable file system that is accessible from all nodes
# (like HDFS, S3, Ceph, nfs, ...)
#
# high-availability.storageDir: hdfs:///flink/ha/
# The list of ZooKeeper quorum peers that coordinate the high-availability
# setup. This must be a list of the form:
# "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)
#
high-availability: zookeeper
high-availability.zookeeper.quorum: {{ zookeeper_servers }}
high-availability.zookeeper.path.root: /flink
high-availability.zookeeper.client.connection-timeout: 150000
high-availability.zookeeper.client.max-retry-attempts: 10
high-availability.zookeeper.client.retry-wait: 10000
high-availability.zookeeper.client.session-timeout: 240000
high-availability.cluster-id: /flink_cluster
# important: customize per cluster
high-availability.storageDir: hdfs://{{ groups.hadoop[0] }}:9000/flink/recover
#high-availability.storageDir: hdfs:///flink/recover
heartbeat.timeout: 180000
heartbeat.interval: 20000
akka.ask.timeout: 300 s
# ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
# It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)
# The default value is "open" and it can be changed to "creator" if ZK security is enabled
#
# high-availability.zookeeper.client.acl: open
#==============================================================================
# Fault tolerance and checkpointing
#==============================================================================
# The backend that will be used to store operator state checkpoints if
# checkpointing is enabled.
#
# Supported backends are 'jobmanager', 'filesystem', 'rocksdb', or the
# <class-name-of-factory>.
#
# state.backend: filesystem
# Directory for checkpoints filesystem, when using any of the default bundled
# state backends.
#
# state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints
# Default target directory for savepoints, optional.
#
# state.savepoints.dir: hdfs://namenode-host:port/flink-savepoints
# Flag to enable/disable incremental checkpoints for backends that
# support incremental checkpoints (like the RocksDB state backend).
#
# state.backend.incremental: false
# The failover strategy, i.e., how the job computation recovers from task failures.
# Only restart tasks that may have been affected by the task failure, which typically includes
# downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.
jobmanager.execution.failover-strategy: region
rest.port: {{ flink_web_port }}
restart-strategy: fixed-delay
restart-strategy.fixed-delay.attempts: 100
restart-strategy.fixed-delay.delay: 10 s
jobmanager.web.upload.dir: {{ install_path }}/{{ flink_version }}/flink-web
#==============================================================================
# Rest & web frontend
#==============================================================================
# The port to which the REST client connects to. If rest.bind-port has
# not been specified, then the server will bind to this port as well.
#
#rest.port: 8092
# The address to which the REST client will connect to
#
#rest.address: 0.0.0.0
# Port range for the REST and web server to bind to.
#
#rest.bind-port: 8080-8090
# The address that the REST & web server binds to
#
#rest.bind-address: 0.0.0.0
# Flag to specify whether job submission is enabled from the web-based
# runtime monitor. Uncomment to disable.
#web.submit.enable: false
#==============================================================================
# Advanced
#==============================================================================
# Override the directories for temporary files. If not specified, the
# system-specific Java temporary directory (java.io.tmpdir property) is taken.
#
# For framework setups on Yarn or Mesos, Flink will automatically pick up the
# containers' temp directories without any need for configuration.
#
# Add a delimited list for multiple directories, using the system directory
# delimiter (colon ':' on unix) or a comma, e.g.:
# /data1/tmp:/data2/tmp:/data3/tmp
#
# Note: Each directory entry is read from and written to by a different I/O
# thread. You can include the same directory multiple times in order to create
# multiple I/O threads against that directory. This is for example relevant for
# high-throughput RAIDs.
#
# io.tmp.dirs: /tmp
# The classloading resolve order. Possible values are 'child-first' (Flink's default)
# and 'parent-first' (Java's default).
#
# Child first classloading allows users to use different dependency/library
# versions in their application than those in the classpath. Switching back
# to 'parent-first' may help with debugging dependency issues.
#
# classloader.resolve-order: child-first
# The amount of memory going to the network stack. These numbers usually need
# no tuning. Adjusting them may be necessary in case of an "Insufficient number
# of network buffers" error. The default min is 64MB, the default max is 1GB.
#
# taskmanager.memory.network.fraction: 0.1
# taskmanager.memory.network.min: 64mb
# taskmanager.memory.network.max: 1gb
#==============================================================================
# Flink Cluster Security Configuration
#==============================================================================
# Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -
# may be enabled in four steps:
# 1. configure the local krb5.conf file
# 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)
# 3. make the credentials available to various JAAS login contexts
# 4. configure the connector to use JAAS/SASL
# The below configure how Kerberos credentials are provided. A keytab will be used instead of
# a ticket cache if the keytab path and principal are set.
# security.kerberos.login.use-ticket-cache: true
# security.kerberos.login.keytab: /path/to/kerberos/keytab
# security.kerberos.login.principal: flink-user
# The configuration below defines which JAAS login contexts
# security.kerberos.login.contexts: Client,KafkaClient
#==============================================================================
# ZK Security Configuration
#==============================================================================
# Below configurations are applicable if ZK ensemble is configured for security
# Override below configuration to provide custom ZK service name if configured
# zookeeper.sasl.service-name: zookeeper
# The configuration below must match one of the values set in "security.kerberos.login.contexts"
# zookeeper.sasl.login-context-name: Client
#==============================================================================
# HistoryServer
#==============================================================================
# The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)
# Directory to upload completed jobs to. Add this directory to the list of
# monitored directories of the HistoryServer as well (see below).
#jobmanager.archive.fs.dir: hdfs:///completed-jobs/
# The address under which the web-based HistoryServer listens.
#historyserver.web.address: 0.0.0.0
# The port under which the web-based HistoryServer listens.
#historyserver.web.port: 8082
# Comma separated list of directories to monitor for completed jobs.
#historyserver.archive.fs.dir: hdfs:///completed-jobs/
#historyserver.archive.fs.dir: /home/bigdata/flink-1.13.1/completed-jobs/
# Interval in milliseconds for refreshing the monitored directories.
#historyserver.archive.fs.refresh-interval: 10000

42
parcels/roles/components/flink/templates/keepflinkalive.j2 Executable file
View File

@@ -0,0 +1,42 @@
#!/bin/bash
#
# netconsole This loads the netconsole module with the configured parameters.
#
# chkconfig:123456 30 70
# description: keepkafkaalive
source /etc/profile
PRO_NAME=keepflinkalive
case $1 in
start)
keepnum=`ps -ef | grep dae-flink.sh | grep -v grep | wc -l`
if [ $keepnum -eq "0" ];then
nohup {{ install_path }}/{{ flink_version }}/bin/dae-flink.sh > /dev/null 2>&1 &
fi
;;
stop)
keeppid=`ps -ef | grep dae-flink.sh | grep -v grep | awk '{print $2}'`
if [ $keeppid ];then
echo "守护进程PID：$keeppid"
kill -9 $keeppid
sleep 1
{{ install_path }}/{{ flink_version }}/bin/stop-cluster.sh
fi
;;
status)
num=`jps | grep -w flink | grep -v grep |wc -l`
if [ "$num" -eq "1" ];then
echo "flink进程已启动"
else
echo "flink进程未启动"
fi
;;
* )
echo "use $0 [start|stop|status]"
;;
esac

0
parcels/roles/components/flink/templates/masters.j2 Normal file
View File

17
parcels/roles/components/flink/templates/set_flink_env.sh.j2 Executable file
View File

@@ -0,0 +1,17 @@
#!/bin/bash
num=`cat /etc/profile | grep flink | wc -l`
if [ $num -eq "0" ];then
echo -e "\n#flink" >> /etc/profile
echo -e "export FLINK_HOME={{ install_path }}/{{ flink_version }}" >> /etc/profile
echo -e "export PATH=\$FLINK_HOME/bin:\$PATH" >> /etc/profile
source /etc/profile
fi
keeppath='/etc/init.d/keepflinkalive'
if [ -x $keeppath ];then
chkconfig --add keepflinkalive
chkconfig keepflinkalive
fi

25
parcels/roles/components/flink/templates/unload_flink.sh.j2 Executable file
View File

@@ -0,0 +1,25 @@
#!/bin/bash
source /etc/profile
function killService(){
keeppath='/etc/init.d/keepflinkalive'
if [ -x $keeppath ];then
service keepflinkalive stop
chkconfig keepkafalive off
systemctl daemon-reload
rm -rf /etc/init.d/keepflinkalive
fi
}
function killPid(){
livenum=`jps -l | egrep -w "flink" | grep -v grep |wc -l`
if [ $livenum -ne 0 ];then
ps aux | grep flink | grep -ivE "grep|unload_flink|ansible" | awk '{print $2}' | xargs kill -9
fi
}
killService
sleep 5
killPid
rm -rf {{ install_path }}/{{ flink_version }}
sleep 5

0
parcels/roles/components/flink/templates/workers.j2 Normal file
View File

5
parcels/roles/components/flink/vars/main.yml Normal file
View File

@@ -0,0 +1,5 @@
#flink版本
flink_version: flink-1.13.1
#web监听端口
flink_web_port: 8080

2
parcels/roles/components/hadoop/tasks/install_hadoop.yml
View File

@@ -34,7 +34,7 @@
force: true
mode: 0755
when: check_out.stdout >= '1'
- name: unload kafka
- name: unload hadoop
shell: cd {{ install_path }} && sh unload_hadoop.sh
when: check_out.stdout >= '1'
- name: Ansible delete {{ install_path }}/unload_hadoop.sh

5
parcels/roles/components/mariadb/install-mariadb-cluster/tasks/install-mariadb.yml
View File

@@ -1,3 +1,8 @@
- name: create {{ install_path }}
file:
state: directory
path: '{{ install_path }}'
- block:
- name: Check if the Mariadb service already exists
shell: rpm -qa | grep MariaDB | wc -l

2042
parcels/roles/init/clickhouse/templates/create_ck_table.sql
View File

File diff suppressed because it is too large Load Diff

4022
parcels/roles/init/clickhouse/templates/create_ck_table.sql_21.06
View File

File diff suppressed because it is too large Load Diff

200
parcels/roles/init/clickhouse/templates/dll.sql
View File

@@ -1,200 +0,0 @@
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log_local on cluster ck_cluster(
common_log_id UInt64,
common_recv_time Int64,
common_entrance_id Int64,
common_device_id String,
common_link_id Int64,
common_policy_id Int64,
common_user_region String,
ad_method String,
ad_protocol String,
common_address_type Int64,
ad_target_ip String,
ad_target_port String,
ad_cc_target_url String,
ad_target_ip_location String,
ad_target_ip_asn String,
ad_claimed_src_ip_profile_id Int64,
ad_reflector_profile_id Int64,
ad_sent_pkt_num Int64,
ad_sent_byte_num Int64,
ad_cc_initiate_connection_num Int64,
ad_cc_established_connection_num Int64,
ad_cc_rejected_connection_num Int64,
ad_generate_time Int64
)
ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time);
create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log on cluster ck_query(
common_log_id UInt64,
common_recv_time Int64,
common_entrance_id Int64,
common_device_id String,
common_link_id Int64,
common_policy_id Int64,
common_user_region String,
ad_method String,
ad_protocol String,
common_address_type Int64,
ad_target_ip String,
ad_target_port String,
ad_cc_target_url String,
ad_target_ip_location String,
ad_target_ip_asn String,
ad_claimed_src_ip_profile_id Int64,
ad_reflector_profile_id Int64,
ad_sent_pkt_num Int64,
ad_sent_byte_num Int64,
ad_cc_initiate_connection_num Int64,
ad_cc_established_connection_num Int64,
ad_cc_rejected_connection_num Int64,
ad_generate_time Int64
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_log_local,rand());
create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log on cluster ck_cluster(
common_log_id UInt64,
common_recv_time Int64,
common_entrance_id Int64,
common_device_id String,
common_link_id Int64,
common_policy_id Int64,
common_user_region String,
ad_method String,
ad_protocol String,
common_address_type Int64,
ad_target_ip String,
ad_target_port String,
ad_cc_target_url String,
ad_target_ip_location String,
ad_target_ip_asn String,
ad_claimed_src_ip_profile_id Int64,
ad_reflector_profile_id Int64,
ad_sent_pkt_num Int64,
ad_sent_byte_num Int64,
ad_cc_initiate_connection_num Int64,
ad_cc_established_connection_num Int64,
ad_cc_rejected_connection_num Int64,
ad_generate_time Int64
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_log_local,rand());
CREATE DICTIONARY IF NOT EXISTS cdn on cluster ck_cluster (
cdn_id UInt64,
domain String,
cname String) PRIMARY KEY cdn_id SOURCE(MYSQL(PORT 3306 USER 'root' PASSWORD 'bifang!@#' REPLICA (HOST '192.168.44.71' PRIORITY 1) DB 'tsg-bifang' TABLE 'tsg_cdn_domain_info')) LIFETIME(MIN 300 MAX 400) LAYOUT(FLAT());
create table IF NOT EXISTS tsg_galaxy_v3.cdn_dic on cluster ck_cluster (`cdn_id` UInt64,
`domain` String,
`cname` String) Engine = Dictionary(cdn);
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local on cluster ck_cluster(
stat_time Int64,
policy_id Int64,
domain String,
ip_list AggregateFunction(groupUniqArray,String),
cdn_list AggregateFunction(groupUniqArray,String),
protocol_type_list AggregateFunction(groupUniqArray,String),
port_list AggregateFunction(groupUniqArray,Int64)
) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,domain,stat_time) SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.security_website_domain_info_local (
stat_time UInt32,
policy_id Int64,
domain String,
ip_list AggregateFunction(groupUniqArray,
String),
cdn_list AggregateFunction(groupUniqArray,
String),
protocol_type_list AggregateFunction(groupUniqArray,
String),
port_list AggregateFunction(groupUniqArray,
Int64)) AS SELECT toUnixTimestamp(toStartOfDay(toDate(common_recv_time))) AS stat_time,
common_policy_id AS policy_id,
http_domain AS domain,
groupUniqArrayState(common_server_ip) AS ip_list,
groupUniqArrayState(cc.domain) AS cdn_list,
groupUniqArrayState(common_schema_type) AS protocol_type_list,
groupUniqArrayState(common_server_port) AS port_list FROM (SELECT common_recv_time,
common_policy_id,
http_domain,
common_server_ip,
arrayJoin(splitByChar(';',
replaceAll(ssl_san,
'*',
''))) AS san,
common_schema_type,
common_server_port FROM tsg_galaxy_v3.security_event_log_local) AS sell INNER JOIN tsg_galaxy_v3.cdn_dic AS cc ON sell.san = cc.domain GROUP BY toStartOfDay(toDate(common_recv_time)),
common_policy_id,
http_domain;
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local on cluster ck_cluster (
stat_time Int64,
policy_id Int64,
ip String,
domain_list AggregateFunction(groupUniqArray,String),
port_list AggregateFunction(groupUniqArray,Int64)
) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,ip,stat_time) SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.security_ip_info_local AS
SELECT
toUnixTimestamp(toStartOfDay(toDate(common_recv_time))) as stat_time,
common_policy_id as policy_id,
common_server_ip as ip,
groupUniqArrayState(http_domain) as domain_list ,
groupUniqArrayState(common_server_port) as port_list
FROM
tsg_galaxy_v3.security_event_log_local
group BY
toStartOfDay(toDate(common_recv_time)),
common_policy_id,
common_server_ip;
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info_local on cluster ck_cluster(
stat_time Int64,
policy_id Int64,
ip_list AggregateFunction(groupUniqArray,String)
) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,stat_time) SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.proxy_ip_info_local (
stat_time UInt32,
policy_id Int64,
ip_list AggregateFunction(groupUniqArray,
String)) AS SELECT toUnixTimestamp(toStartOfMinute(toDateTime(common_recv_time))) AS stat_time,
common_policy_id AS policy_id,
groupUniqArrayState(common_client_ip) AS ip_list FROM tsg_galaxy_v3.proxy_event_log_local GROUP BY toStartOfMinute(toDateTime(common_recv_time)),
common_policy_id;
create table IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info on cluster ck_query
(
stat_time Int64,
policy_id Int64,
domain String,
ip_list AggregateFunction(groupUniqArray,String),
cdn_list AggregateFunction(groupUniqArray,String),
protocol_type_list AggregateFunction(groupUniqArray,String),
port_list AggregateFunction(groupUniqArray,Int64)
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_website_domain_info_local,rand());
create table IF NOT EXISTS tsg_galaxy_v3.security_ip_info on cluster ck_query (
stat_time Int64,
policy_id Int64,
ip String,
domain_list AggregateFunction(groupUniqArray,String),
port_list AggregateFunction(groupUniqArray,Int64)
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_ip_info_local,rand());
create table IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info on cluster ck_query (
stat_time Int64,
policy_id Int64,
ip_list AggregateFunction(groupUniqArray,String)
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_ip_info_local,rand());

57
parcels/roles/init/dos-baseline/tasks/init_dos.yml Normal file
View File

@@ -0,0 +1,57 @@
- name: check jdk 1.8_73
shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l
ignore_errors: false
register: jdk_out
- name: To terminate execution
fail:
msg: "检测到目标集群内存在JDK未安装节点，请检查"
when: jdk_out.stdout != '2'
run_once: true
delegate_to: 127.0.0.1
- name: create dos-baseline package path:{{ install_path }}
file:
state: directory
path: '{{ install_path }}'
- name: get zookeeper_servers to ansible variable
set_fact: zookeeper_servers="{{groups.zookeeper[0]}},{{groups.zookeeper[1]}},{{groups.zookeeper[2]}}"
when: '(groups.zookeeper|length) == 3'
- name: get zookeeper_servers to ansible variable
set_fact: zookeeper_servers="{{groups.zookeeper[0]}},{{groups.zookeeper[1]}},{{groups.zookeeper[2]}},{{groups.zookeeper[3]}},{{groups.zookeeper[4]}}"
when: '(groups.zookeeper|length) >= 5'
- name: master_ip from ansible variable
set_fact: master_ip="{{groups.dos_baseline[0]}}"
- name: copy program pkg to destination:{{ master_ip }}
unarchive:
src: "{{ package_path }}/dos-baseline.tar.gz"
dest: "{{ install_path }}"
force: true
backup: yes
delegate_to: "{{ master_ip }}"
- name: copy configuration
template:
src: 'application.properties.j2'
dest: '{{ install_path }}/dos-baseline/application.properties'
mode: 0644
backup: yes
delegate_to: "{{ master_ip }}"
#- name: start dos-baseline
# shell: sh {{ install_path }}/dos-baseline/start.sh
# run_once: true
# delegate_to: "{{ master_ip }}"
- name: add crontab task to schedule dos-baseline
cron:
name: 'dos-baseline'
hour: "3"
weekday: "1"
job: '/bin/sh {{ install_path }}/dos-baseline/start.sh'
user: root
delegate_to: "{{ master_ip }}"

1
parcels/roles/init/dos-baseline/tasks/main.yml Normal file
View File

@@ -0,0 +1 @@
- include: init_dos.yml

78
parcels/roles/init/dos-baseline/templates/application.properties.j2 Normal file
View File

@@ -0,0 +1,78 @@
##########################################
############## 数据库配置 ###############
##########################################
#Druid配置
druid.url=jdbc:avatica:remote:url=http://{{ gateway_keepalive_host }}:8089/druid/v2/sql/avatica/
druid.driver=org.apache.calcite.avatica.remote.Driver
druid.table=traffic_top_destination_ip_metrics_log
#HBase配置
hbase.table=ddos_traffic_baselines
hbase.zookeeper.quorum={{ zookeeper_servers }}
hbase.zookeeper.client.port=2181
##########################################
############## Druid数据读取 ###############
##########################################
#读取druid时间范围方式，
# 0：读取默认范围天数read.historical.days；
# 1：指定时间范围
read.druid.time.limit.type=0
#07-05
read.druid.min.time=1625414400000
#07-08
read.druid.max.time=1625673600000
#Druid字段映射
druid.attacktype.tcpsynflood=TCP SYN Flood
druid.attacktype.udpflood=UDP Flood
druid.attacktype.icmpflood=ICMP Flood
druid.attacktype.dnsamplification=DNS Amplification
druid.columnname.serverip=destination_ip
druid.columnname.attacktype=attack_type
druid.columnname.recvtime=__time
druid.columnname.partition.num=partition_num
baseline.metric.type=session_rate
#Hbase字段映射
hbase.baseline.generation.type.suffix=baseline_type
hbase.baseline.zero.replace.value.suffix=default_value
#数据情况
#读取历史N天数据，最小值为3天（需要判断周期性）
read.historical.days=7
#历史数据汇聚粒度为10分钟
historical.grad=10
# 数据库Time格式
time.format=yyyy-MM-dd HH:mm:ss
##########################################
############ Baseline生成参数 #############
##########################################
baseline.range.days=1
baseline.function=KalmanFilter
baseline.period.correlative.threshold=0.5
baseline.historical.frequency.thread=0.2
baseline.exception.percentile=0.99
baseline.exception.fill.percentile=0.99
baseline.rational.percentile=0.95
#Kalman Filter
baseline.kalman.q=0.1
baseline.kalman.r=0.1
baseline.kalman.p=8
baseline.kalman.m=2
monitor.frequency.bin.num=100
##########################################
################ 并发参数 #################
##########################################
#druid.read.batch.time.grad.hour=4
thread.pool.num=20
#druid分区字段partition_num的最大值为9999
druid.partition.num.max=10000
druid.connection.retry.time.max=10
druid.connection.retry.sleep.time=1000

62
parcels/roles/init/druid/files/druid_segments-tsg-21.05.sql
View File

@@ -1,62 +0,0 @@
/*
Navicat Premium Data Transfer
Source Server : 192.168.44.226
Source Server Type : MariaDB
Source Server Version : 100503
Source Host : 192.168.44.226:3306
Source Schema : druid_33
Target Server Type : MariaDB
Target Server Version : 100503
File Encoding : 65001
Date: 02/04/2021 11:52:27
*/
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for druid_segments
-- ----------------------------
DROP TABLE IF EXISTS `druid_segments`;
CREATE TABLE `druid_segments` (
`id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`dataSource` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
`created_date` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`start` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`end` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`partitioned` tinyint(1) NOT NULL,
`version` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
`used` tinyint(1) NOT NULL,
`payload` longblob NOT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `idx_druid_segments_used`(`used`) USING BTREE,
INDEX `idx_druid_segments_datasource_used_end`(`dataSource`, `used`, `end`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Records of druid_segments
-- ----------------------------
INSERT INTO `druid_segments` VALUES ('client_internal_ip_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:39.273Z', 'client_internal_ip_hot_log', '2021-04-02T03:13:40.184Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:39.273Z', 1, 0x7B2264617461536F75726365223A22636C69656E745F696E7465726E616C5F69705F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A33392E3237335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F636C69656E745F696E7465726E616C5F69705F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F33392E3237335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F636C69656E745F69702C636F6D6D6F6E5F696E7465726E616C5F69702C636F6D6D6F6E5F6C345F70726F746F636F6C2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323630372C226964656E746966696572223A22636C69656E745F696E7465726E616C5F69705F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A33392E3237335A227D);
INSERT INTO `druid_segments` VALUES ('proxy_event_hits_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:41.641Z', 'proxy_event_hits_log', '2021-04-02T03:13:42.617Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:41.641Z', 1, 0x7B2264617461536F75726365223A2270726F78795F6576656E745F686974735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34312E3634315A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F70726F78795F6576656E745F686974735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34312E3634315A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F756E7472792C6973702C7375625F616374696F6E2C6C6F636174696F6E2C616374696F6E2C656E7472616E63655F69642C686974732C706F6C6963795F69642C6332735F627974655F6E756D2C7332635F627974655F6E756D222C226D657472696373223A2269705F6F626A656374222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A333239332C226964656E746966696572223A2270726F78795F6576656E745F686974735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34312E3634315A227D);
INSERT INTO `druid_segments` VALUES ('proxy_event_urls_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:43.849Z', 'proxy_event_urls_hot_log', '2021-04-02T03:13:44.736Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:43.849Z', 1, 0x7B2264617461536F75726365223A2270726F78795F6576656E745F75726C735F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34332E3834395A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F70726F78795F6576656E745F75726C735F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34332E3834395A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A2270726F78795F6576656E745F75726C735F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34332E3834395A227D);
INSERT INTO `druid_segments` VALUES ('security_event_hits_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:46.392Z', 'security_event_hits_log', '2021-04-02T03:13:47.468Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:46.392Z', 1, 0x7B2264617461536F75726365223A2273656375726974795F6576656E745F686974735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34362E3339325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F73656375726974795F6576656E745F686974735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34362E3339325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C616374696F6E2C656E7472616E63655F69642C686974732C706F6C6963795F69642C6332735F627974655F6E756D2C7332635F627974655F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313939332C226964656E746966696572223A2273656375726974795F6576656E745F686974735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34362E3339325A227D);
INSERT INTO `druid_segments` VALUES ('security_event_urls_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:48.458Z', 'security_event_urls_hot_log', '2021-04-02T03:13:49.325Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:48.458Z', 1, 0x7B2264617461536F75726365223A2273656375726974795F6576656E745F75726C735F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34382E3435385A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F73656375726974795F6576656E745F75726C735F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34382E3435385A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A2273656375726974795F6576656E745F75726C735F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34382E3435385A227D);
INSERT INTO `druid_segments` VALUES ('server_external_ip_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:50.092Z', 'server_external_ip_hot_log', '2021-04-02T03:13:51.021Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:50.092Z', 1, 0x7B2264617461536F75726365223A227365727665725F65787465726E616C5F69705F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35302E3039325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F7365727665725F65787465726E616C5F69705F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35302E3039325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F7365727665725F69702C636F6D6D6F6E5F65787465726E616C5F69702C636F6D6D6F6E5F6C345F70726F746F636F6C2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323630372C226964656E746966696572223A227365727665725F65787465726E616C5F69705F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35302E3039325A227D);
INSERT INTO `druid_segments` VALUES ('sys_storage_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:52.202Z', 'sys_storage_log', '2021-04-02T03:13:53.057Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:52.202Z', 1, 0x7B2264617461536F75726365223A227379735F73746F726167655F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35322E3230325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F7379735F73746F726167655F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35322E3230325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226C6F675F747970652C646174615F63656E7465722C6167677265676174655F73697A652C6C6173745F73746F726167652C6D61785F73697A652C757365645F73697A65222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313938322C226964656E746966696572223A227379735F73746F726167655F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35322E3230325A227D);
INSERT INTO `druid_segments` VALUES ('top_client_ip_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:53.675Z', 'top_client_ip_log', '2021-04-02T03:13:54.681Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:53.675Z', 1, 0x7B2264617461536F75726365223A22746F705F636C69656E745F69705F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35332E3637355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F636C69656E745F69705F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35332E3637355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22736F757263652C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F636C69656E745F69705F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35332E3637355A227D);
INSERT INTO `druid_segments` VALUES ('top_external_host_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:55.845Z', 'top_external_host_log', '2021-04-02T03:13:57.329Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:55.845Z', 1, 0x7B2264617461536F75726365223A22746F705F65787465726E616C5F686F73745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35352E3834355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F65787465726E616C5F686F73745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35352E3834355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2264657374696E6174696F6E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139322C226964656E746966696572223A22746F705F65787465726E616C5F686F73745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35352E3834355A227D);
INSERT INTO `druid_segments` VALUES ('top_internal_host_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:58.433Z', 'top_internal_host_log', '2021-04-02T03:13:59.446Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:58.433Z', 1, 0x7B2264617461536F75726365223A22746F705F696E7465726E616C5F686F73745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35382E3433335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F696E7465726E616C5F686F73745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35382E3433335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22736F757263652C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F696E7465726E616C5F686F73745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35382E3433335A227D);
INSERT INTO `druid_segments` VALUES ('top_server_ip_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:59.654Z', 'top_server_ip_log', '2021-04-02T03:14:00.955Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:59.654Z', 1, 0x7B2264617461536F75726365223A22746F705F7365727665725F69705F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35392E3635345A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F69705F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35392E3635345A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2264657374696E6174696F6E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139322C226964656E746966696572223A22746F705F7365727665725F69705F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35392E3635345A227D);
INSERT INTO `druid_segments` VALUES ('top_urls_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:01.484Z', 'top_urls_log', '2021-04-02T03:14:02.269Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:01.484Z', 1, 0x7B2264617461536F75726365223A22746F705F75726C735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30312E3438345A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F75726C735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30312E3438345A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A22746F705F75726C735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30312E3438345A227D);
INSERT INTO `druid_segments` VALUES ('top_user_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:04.063Z', 'top_user_log', '2021-04-02T03:14:04.955Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:04.063Z', 1, 0x7B2264617461536F75726365223A22746F705F757365725F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30342E3036335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F757365725F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30342E3036335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22737562736372696265725F69642C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139382C226964656E746966696572223A22746F705F757365725F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30342E3036335A227D);
INSERT INTO `druid_segments` VALUES ('top_website_domain_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:06.100Z', 'top_website_domain_log', '2021-04-02T03:14:07.150Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:06.100Z', 1, 0x7B2264617461536F75726365223A22746F705F776562736974655F646F6D61696E5F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30362E3130305A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F776562736974655F646F6D61696E5F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30362E3130305A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22646F6D61696E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F776562736974655F646F6D61696E5F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30362E3130305A227D);
INSERT INTO `druid_segments` VALUES ('traffic_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:08.462Z', 'traffic_metrics_log', '2021-04-02T03:14:09.342Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:08.462Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6D6574726963735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30382E3436325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F6D6574726963735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30382E3436325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226465766963655F69642C616C6C6F775F636F6E6E5F6E756D2C616C6C6F775F696E5F62797465732C616C6C6F775F696E5F7061636B6574732C616C6C6F775F6F75745F62797465732C616C6C6F775F6F75745F7061636B6574732C636C6F73655F636F6E6E5F6E756D2C64656661756C745F636F6E6E5F6E756D2C64656661756C745F696E5F62797465732C64656661756C745F696E5F7061636B6574732C64656661756C745F6F75745F62797465732C64656661756C745F6F75745F7061636B6574732C64656E795F636F6E6E5F6E756D2C64656E795F696E5F62797465732C64656E795F696E5F7061636B6574732C64656E795F6F75745F62797465732C64656E795F6F75745F7061636B6574732C65737461626C69736865645F636F6E6E5F6E756D2C696E746572636570745F636F6E6E5F6E756D2C696E746572636570745F696E5F62797465732C696E746572636570745F696E5F7061636B6574732C696E746572636570745F6F75745F62797465732C696E746572636570745F6F75745F7061636B6574732C6D6F6E69746F725F636F6E6E5F6E756D2C6D6F6E69746F725F696E5F62797465732C6D6F6E69746F725F696E5F7061636B6574732C6D6F6E69746F725F6F75745F62797465732C6D6F6E69746F725F6F75745F7061636B6574732C6E65775F636F6E6E5F6E756D2C746F74616C5F696E5F62797465732C746F74616C5F696E5F7061636B6574732C746F74616C5F6F75745F62797465732C656E7472616E63655F69642C746F74616C5F6F75745F7061636B6574732C70696E6E696E675F6E756D2C6E6F745F70696E6E696E675F6E756D2C6D617962655F70696E6E696E675F6E756D2C616C6572745F62797465732C696E735F62797465732C68696A6B5F62797465732C61645F7265666C656374696F6E5F62797465732C61645F666C6F6F645F62797465732C61645F63635F62797465732C696E7463705F6D6F6E5F6E756D2C696E7463705F616C6C6F775F6E756D2C696E7463705F64656E795F6E756D2C696E7463705F72646972745F6E756D2C696E7463705F7265706C5F6E756D2C696E7463705F68696A6B5F6E756D2C696E7463705F696E735F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A31313430362C226964656E746966696572223A22747261666669635F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30382E3436325A227D);
INSERT INTO `druid_segments` VALUES ('traffic_protocol_stat_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:09.706Z', 'traffic_protocol_stat_log', '2021-04-02T03:14:10.583Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:09.706Z', 1, 0x7B2264617461536F75726365223A22747261666669635F70726F746F636F6C5F737461745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30392E3730365A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F70726F746F636F6C5F737461745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30392E3730365A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C70726F746F636F6C5F69642C646174615F63656E7465722C6332735F706B745F6E756D2C656E7472616E63655F69642C73657373696F6E732C7332635F706B745F6E756D2C6332735F627974655F6E756D2C7332635F627974655F6E756D2C6332735F6970667261675F6E756D2C7332635F6970667261675F6E756D2C6332735F7463705F6C6F73746C656E2C7332635F7463705F6C6F73746C656E2C6332735F7463705F756E6F726465725F6E756D2C7332635F7463705F756E6F726465725F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A343032372C226964656E746966696572223A22747261666669635F70726F746F636F6C5F737461745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30392E3730365A227D);
INSERT INTO `druid_segments` VALUES ('traffic_summary_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:12.193Z', 'traffic_summary_log', '2021-04-02T03:14:13.233Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:12.193Z', 1, 0x7B2264617461536F75726365223A22747261666669635F73756D6D6172795F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31322E3139335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F73756D6D6172795F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31322E3139335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C736368656D615F747970652C646174615F63656E7465722C6332735F706B745F6E756D2C656E7472616E63655F69642C73657373696F6E732C7332635F706B745F6E756D2C6332735F627974655F6E756D2C7332635F627974655F6E756D2C6F6E655F73696465645F636F6E6E656374696F6E732C756E63617465676F72697A65645F62797465732C667261676D656E746174696F6E5F7061636B6574732C73657175656E63655F6761705F6C6F73732C756E6F726465725F7061636B657473222C226D657472696373223A2269705F6F626A656374222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A343133342C226964656E746966696572223A22747261666669635F73756D6D6172795F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31322E3139335A227D);
INSERT INTO `druid_segments` VALUES ('user_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:14.350Z', 'user_hot_log', '2021-04-02T03:14:15.137Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:14.350Z', 1, 0x7B2264617461536F75726365223A22757365725F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31342E3335305A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F757365725F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31342E3335305A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F737562736372696265725F69642C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313838332C226964656E746966696572223A22757365725F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31342E3335305A227D);
INSERT INTO `druid_segments` VALUES ('website_domain_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:16.425Z', 'website_domain_hot_log', '2021-04-02T03:14:17.174Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:16.425Z', 1, 0x7B2264617461536F75726365223A22776562736974655F646F6D61696E5F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31362E3432355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F776562736974655F646F6D61696E5F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31362E3432355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22646F6D61696E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313834312C226964656E746966696572223A22776562736974655F646F6D61696E5F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31362E3432355A227D);
SET FOREIGN_KEY_CHECKS = 1;

2
parcels/roles/init/druid/files/druid_segments-tsg3.0.sql
View File

@@ -60,4 +60,6 @@ INSERT INTO `druid_segments` VALUES ('user_hot_log_3000-01-02T00:00:00.000Z_3000
INSERT INTO `druid_segments` VALUES ('website_domain_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:16.425Z', 'website_domain_hot_log', '2021-04-02T03:14:17.174Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:16.425Z', 1, 0x7B2264617461536F75726365223A22776562736974655F646F6D61696E5F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31362E3432355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F776562736974655F646F6D61696E5F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31362E3432355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22646F6D61696E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313834312C226964656E746966696572223A22776562736974655F646F6D61696E5F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31362E3432355A227D);
INSERT INTO `druid_segments` VALUES ('traffic_app_stat_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-07-02T02:43:09.615Z', 'traffic_app_stat_log', '2021-07-02T02:43:10.572Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-07-02T02:43:09.615Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6170705F737461745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30372D30325430323A34333A30392E3631355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F6170705F737461745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30372D30325430325F34335F30392E3631355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226170705F6E616D652C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C7375625F6170705F6E616D65222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139352C226964656E746966696572223A22747261666669635F6170705F737461745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30372D30325430323A34333A30392E3631355A227D);
INSERT INTO `druid_segments`(`id`, `dataSource`, `created_date`, `start`, `end`, `partitioned`, `version`, `used`, `payload`) VALUES ('traffic_top_destination_ip_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-08-11T10:21:56.486Z', 'traffic_top_destination_ip_metrics_log', '2021-08-11T10:21:57.448Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-08-11T10:21:56.486Z', 1, 0x7B2264617461536F75726365223A22747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30382D31315431303A32313A35362E3438365A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30382D31315431305F32315F35362E3438365A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2261747461636B5F747970652C636F6D6D6F6E5F646174615F63656E7465722C636F6D6D6F6E5F736C65645F69702C64657374696E6174696F6E5F69702C6269745F726174652C7061636B65745F726174652C706172746974696F6E5F6E756D2C73657373696F6E5F72617465222C226D657472696373223A2269705F6F626A656374222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A333031302C226964656E746966696572223A22747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30382D31315431303A32313A35362E3438365A227D);
SET FOREIGN_KEY_CHECKS = 1;

15
parcels/roles/init/druid/files/druid_topology/initDruidData/initData.sh
View File

@@ -1,15 +0,0 @@
#!/bin/bash
PWD=`pwd`
task_name=`cat ${PWD}/tasks.txt`
#overlord节点IP
overlord_ip=$1
for var in ${task_name[@]};
do
curl -X 'POST' -H 'Content-Type:application/json' -d @${var} http://${overlord_ip}:8081/druid/indexer/v1/task
echo "'${var}' 数据初始化成功"
sleep 2
done

70
parcels/roles/init/druid/files/druid_topology/initDruidData/init_client_internal_ip_hot_log.json
View File

@@ -1,70 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"common_client_ip\":\" \",\"common_internal_ip\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"common_l4_protocol\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "client_internal_ip_hot_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "common_client_ip",
"type": "string"
},
{
"name": "common_internal_ip",
"type": "string"
},
{
"name": "common_l4_protocol",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
}
]
}
}
}
}
}
}

85
parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_hits_log.json
View File

@@ -1,85 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"country\":\" \",\"action\":1024,\"c2s_byte_num\":0,\"s2c_byte_num\":0,\"entrance_id\":0,\"policy_id\":0,\"location\":\" \",\"hits\":0,\"sub_action\":\" \",\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "proxy_event_hits_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": true
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "country",
"type": "string"
},
{
"name": "isp",
"type": "string"
},
{
"name": "sub_action",
"type": "string"
},
{
"name": "location",
"type": "string"
},
{
"name": "action",
"type": "long"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "hits",
"type": "long"
},
{
"name": "policy_id",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
}
]
}
}
},
"metricsSpec": [
{
"type": "HLLSketchBuild",
"name": "ip_object",
"fieldName": "ips"
}
]
}
}
}

46
parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_urls_hot_log.json
View File

@@ -1,46 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"url\":\" \",\"session_num\":0,\"device_id\":\" \",\"ips\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "proxy_event_urls_hot_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "url",
"type": "string"
},
{
"name": "session_num",
"type": "long"
}
]
}
}
}
}
}
}

66
parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_hits_log.json
View File

@@ -1,66 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"country\":\" \",\"action\":1024,\"c2s_byte_num\":0,\"s2c_byte_num\":0,\"entrance_id\":0,\"policy_id\":0,\"location\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "security_event_hits_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "isp",
"type": "string"
},
{
"name": "action",
"type": "long"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "hits",
"type": "long"
},
{
"name": "policy_id",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
}
]
}
}
}
}
}
}

46
parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_urls_hot_log.json
View File

@@ -1,46 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"url\":\" \",\"session_num\":0,\"device_id\":\" \",\"ips\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "security_event_urls_hot_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "url",
"type": "string"
},
{
"name": "session_num",
"type": "long"
}
]
}
}
}
}
}
}

70
parcels/roles/init/druid/files/druid_topology/initDruidData/init_server_external_ip_hot_log.json
View File

@@ -1,70 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"common_server_ip\":\" \",\"common_external_ip\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"common_l4_protocol\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "server_external_ip_hot_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "common_server_ip",
"type": "string"
},
{
"name": "common_external_ip",
"type": "string"
},
{
"name": "common_l4_protocol",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
}
]
}
}
}
}
}
}

62
parcels/roles/init/druid/files/druid_topology/initDruidData/init_sys_storage_log.json
View File

@@ -1,62 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"log_type\":\" \",\"data_center\":\" \",\"aggregate_size\":0,\"first_storage\":0,\"max_size\":0,\"stored_days\":0,\"used_size\":0,\"location\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "sys_storage_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "log_type",
"type": "string"
},
{
"name": "data_center",
"type": "string"
},
{
"name": "aggregate_size",
"type": "long"
},
{
"name": "last_storage",
"type": "long"
},
{
"name": "max_size",
"type": "long"
},
{
"name": "used_size",
"type": "long"
}
]
}
}
}
}
}
}

66
parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_client_ip_log.json
View File

@@ -1,66 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"source\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "top_client_ip_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "source",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
}
]
}
}
}
}
}
}

66
parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_external_host_log.json
View File

@@ -1,66 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"destination\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "top_external_host_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "destination",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
}
]
}
}
}
}
}
}

66
parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_internal_host_log.json
View File

@@ -1,66 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"source\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "top_internal_host_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "source",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
}
]
}
}
}
}
}
}

66
parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_server_ip_log.json
View File

@@ -1,66 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"destination\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "top_server_ip_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "destination",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
}
]
}
}
}
}
}
}

46
parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_urls_log.json
View File

@@ -1,46 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"url\":\" \",\"session_num\":0,\"device_id\":\" \",\"ips\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "top_urls_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "url",
"type": "string"
},
{
"name": "session_num",
"type": "long"
}
]
}
}
}
}
}
}

66
parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_user_log.json
View File

@@ -1,66 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"subscriber_id\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "top_user_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "subscriber_id",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
}
]
}
}
}
}
}
}

66
parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_website_domain_log.json
View File

@@ -1,66 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"domain\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "top_website_domain_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "domain",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
}
]
}
}
}
}
}
}

67
parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_app_stat_log.json
View File

@@ -1,67 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"app_name\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"sub_app_name\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "traffic_app_stat_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "app_name",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "sub_app_name",
"type": "string"
}
]
}
}
}
}
}
}

88
parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_metrics_log.json
View File

@@ -1,88 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"device_id\":\" \",\"allow_conn_num\":0,\"allow_in_bytes\":0,\"allow_in_packets\":0,\"allow_out_bytes\":0,\"allow_out_packets\":0,\"close_conn_num\":0,\"default_conn_num\":0,\"default_in_bytes\":0,\"default_in_packets\":0,\"default_out_bytes\":0,\"default_out_packets\":0,\"deny_conn_num\":0,\"deny_in_bytes\":0,\"deny_in_packets\":0,\"deny_out_bytes\":0,\"deny_out_packets\":0,\"established_conn_num\":0,\"intercept_conn_num\":0,\"intercept_in_bytes\":0,\"intercept_in_packets\":0,\"intercept_out_bytes\":0,\"intercept_out_packets\":0,\"monitor_conn_num\":0,\"monitor_in_bytes\":0,\"monitor_in_packets\":0,\"monitor_out_bytes\":0,\"monitor_out_packets\":0,\"new_conn_num\":0,\"total_in_bytes\":0,\"total_in_packets\":0,\"total_out_bytes\":0,\"total_out_packets\":0,\"entrance_id\":0,\"pinning_num\":0,\"not_pinning_num\":0,\"maybe_pinning_num\":0,\"alert_bytes\":0,\"ins_bytes\":0,\"hijk_bytes\":0,\"reflection_bytes\":0,\"flood_bytes\":0,\"intcp_mon_num\":0,\"intcp_allow_num\":0,\"intcp_deny_num\":0,\"intcp_rdirt_num\":0,\"intcp_repl_num\":0,\"intcp_hijk_num\":0,\"intcp_ins_num\":0,\"ad_reflection_bytes\":0,\"ad_flood_bytes\":0,\"ad_cc_bytes\":0}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "traffic_metrics_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{"name": "device_id","type": "string"},
{"name": "allow_conn_num","type": "long"},
{"name": "allow_in_bytes","type": "long"},
{"name": "allow_in_packets","type": "long"},
{"name": "allow_out_bytes","type": "long"},
{"name": "allow_out_packets","type": "long"},
{"name": "close_conn_num","type": "long"},
{"name": "default_conn_num","type": "long"},
{"name": "default_in_bytes","type": "long"},
{"name": "default_in_packets","type": "long"},
{"name": "default_out_bytes","type": "long"},
{"name": "default_out_packets","type": "long"},
{"name": "deny_conn_num","type": "long"},
{"name": "deny_in_bytes","type": "long"},
{"name": "deny_in_packets","type": "long"},
{"name": "deny_out_bytes","type": "long"},
{"name": "deny_out_packets","type": "long"},
{"name": "established_conn_num","type": "long"},
{"name": "intercept_conn_num","type": "long"},
{"name": "intercept_in_bytes","type": "long"},
{"name": "intercept_in_packets","type": "long"},
{"name": "intercept_out_bytes","type": "long"},
{"name": "intercept_out_packets","type": "long"},
{"name": "monitor_conn_num","type": "long"},
{"name": "monitor_in_bytes","type": "long"},
{"name": "monitor_in_packets","type": "long"},
{"name": "monitor_out_bytes","type": "long"},
{"name": "monitor_out_packets","type": "long"},
{"name": "new_conn_num","type": "long"},
{"name": "total_in_bytes","type": "long"},
{"name": "total_in_packets","type": "long"},
{"name": "total_out_bytes","type": "long"},
{"name": "entrance_id","type": "long"},
{"name": "total_out_packets","type": "long"},
{"name": "pinning_num","type": "long"},
{"name": "not_pinning_num","type": "long"},
{"name": "maybe_pinning_num","type": "long"},
{"name": "alert_bytes","type": "long"},
{"name": "ins_bytes","type": "long"},
{"name": "hijk_bytes","type": "long"},
{"name": "ad_reflection_bytes","type": "long"},
{"name": "ad_flood_bytes","type": "long"},
{"name": "ad_cc_bytes","type": "long"},
{"name": "intcp_mon_num","type": "long"},
{"name": "intcp_allow_num","type": "long"},
{"name": "intcp_deny_num","type": "long"},
{"name": "intcp_rdirt_num","type": "long"},
{"name": "intcp_repl_num","type": "long"},
{"name": "intcp_hijk_num","type": "long"},
{"name": "intcp_ins_num","type": "long"}
]
}
}
}
}
}
}

54
parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_protocol_stat_log.json
View File

@@ -1,54 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"protocol_id\":\" \",\"country\":\" \",\"one_sided_connections\":1024,\"sequence_gap_loss\":0,\"s2c_byte_num\":0,\"entrance_id\":0,\"fragmentation_packets\":0,\"location\":\" \",\"uncategorized_bytes\":0,\"unorder_packets\":0,\"sub_action\":\" \",\"ips\":\" \",\"isp\":\" \",\"schema_type\":\" \",\"data_center\":\" \",\"s2c_pkt_num\":0,\"s2c_pkt_num\":0,\"c2s_ipfrag_num\":0,\"s2c_ipfrag_num\":0,\"c2s_tcp_lostlen\":0,\"s2c_tcp_lostlen\":0,\"c2s_tcp_unorder_num\":0,\"s2c_tcp_unorder_num\":0}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "traffic_protocol_stat_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": true
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{"name": "isp","type": "string"},
{"name": "protocol_id","type": "string"},
{"name": "data_center","type": "string"},
{"name": "c2s_pkt_num","type": "long"},
{"name": "entrance_id","type": "long"},
{"name": "sessions","type": "long"},
{"name": "s2c_pkt_num","type": "long"},
{"name": "c2s_byte_num","type": "long"},
{"name": "s2c_byte_num","type": "long"},
{"name": "c2s_ipfrag_num","type": "long"},
{"name": "s2c_ipfrag_num","type": "long"},
{"name": "c2s_tcp_lostlen","type": "long"},
{"name": "s2c_tcp_lostlen","type": "long"},
{"name": "c2s_tcp_unorder_num","type": "long"},
{"name": "s2c_tcp_unorder_num","type": "long"}
]
}
}
},
"metricsSpec": []
}
}
}

53
parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_summary_log.json
View File

@@ -1,53 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"country\":\" \",\"one_sided_connections\":1024,\"sequence_gap_loss\":0,\"s2c_byte_num\":0,\"entrance_id\":0,\"fragmentation_packets\":0,\"location\":\" \",\"uncategorized_bytes\":0,\"unorder_packets\":0,\"sub_action\":\" \",\"ips\":\" \",\"isp\":\" \",\"schema_type\":\" \",\"data_center\":\" \",\"s2c_pkt_num\":0,\"s2c_pkt_num\":0,\"c2s_ipfrag_num\":0,\"s2c_ipfrag_num\":0,\"c2s_tcp_lostlen\":0,\"s2c_tcp_lostlen\":0,\"c2s_tcp_unorder_num\":0,\"s2c_tcp_unorder_num\":0}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "traffic_summary_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": true
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{"name": "isp","type": "string"},
{"name": "schema_type","type": "string"},
{"name": "data_center","type": "string"},
{"name": "c2s_pkt_num","type": "long"},
{"name": "entrance_id","type": "long"},
{"name": "sessions","type": "long"},
{"name": "s2c_pkt_num","type": "long"},
{"name": "c2s_byte_num","type": "long"},
{"name": "s2c_byte_num","type": "long"},
{"name": "one_sided_connections","type": "long"},
{"name": "uncategorized_bytes","type": "long"},
{"name": "fragmentation_packets","type": "long"},
{"name": "sequence_gap_loss","type": "long"},
{"name": "unorder_packets","type": "long"}
]
}
}
},
"metricsSpec": [{"type": "HLLSketchBuild","name": "ip_object","fieldName": "ips"}]
}
}
}

62
parcels/roles/init/druid/files/druid_topology/initDruidData/init_user_hot_log.json
View File

@@ -1,62 +0,0 @@
{
"type": "index",
"spec":{
"ioConfig": {
"type": "index",
"firehose": {
"type": "inline",
"data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"common_subscriber_id\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}"
}
},
"tuningConfig": {
"type": "index"
},
"dataSchema": {
"dataSource": "user_hot_log",
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": "NONE",
"rollup": false
},
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "test_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
{
"name": "common_subscriber_id",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
}
]
}
}
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More