From 0b1e1543beca9abcf5e9d2c126277db875a7e276 Mon Sep 17 00:00:00 2001 From: qidaijie Date: Thu, 22 Feb 2024 11:00:02 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4tsg-olap=2024.01=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E5=88=9D=E5=A7=8B=E5=8C=96=E5=89=A7=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../initialization/arangodb/tasks/check.yml | 11 + .../initialization/arangodb/tasks/init.yml | 21 + .../initialization/arangodb/tasks/main.yml | 3 + .../arangodb/templates/create_database.sh.j2 | 54 + .../initialization/arangodb/vars/main.yml | 9 + .../clickhouse/files/create_ck_table.sql | 4174 +++++++++++++++++ .../clickhouse/files/system.sql | 9 + .../files/upgrade-sql/update-23.01-ck.sql | 247 + .../files/upgrade-sql/update-23.02-ck.sql | 110 + .../files/upgrade-sql/update-23.03-ck.sql | 47 + .../files/upgrade-sql/update-23.04-ck.sql | 6 + .../files/upgrade-sql/update-23.05-ck.sql | 11 + .../files/upgrade-sql/update-23.06-ck.sql | 159 + .../files/upgrade-sql/update-23.07-ck.sql | 97 + .../files/upgrade-sql/update-23.08-ck.sql | 158 + .../files/upgrade-sql/update-23.09-ck.sql | 131 + .../files/upgrade-sql/update-23.10-ck.sql | 76 + .../initialization/clickhouse/tasks/init.yml | 24 + .../initialization/clickhouse/tasks/main.yml | 5 + .../clickhouse/tasks/upgrate.yml | 25 + .../templates/upgrade-clickhouse-data.sh.j2 | 50 + .../initialization/druid/tasks/backup.yml | 25 + .../initialization/druid/tasks/check.yml | 23 + .../druid/tasks/init-cluster.yml | 115 + .../druid/tasks/init-standalone.yml | 65 + .../roles/initialization/druid/tasks/main.yml | 21 + .../initialization/druid/tasks/uninstall.yml | 30 + .../druid/templates/druid_monitor.sh.j2 | 58 + .../druid/templates/rule/post_rule.sh.j2 | 28 + .../templates/rule/supervisor-manager.j2 | 75 + .../druid/templates/run_druid_monitor.j2 | 3 + .../initialization/druid/templates/s.bak | 124 + .../druid/templates/supervisor-manager.sh.j2 | 103 + ...index_kafka_application_protocol_stat.json | 131 + .../tasks/index_kafka_monitor_rule_hits.json | 83 + .../tasks/index_kafka_object_statistics.json | 95 + .../tasks/index_kafka_proxy_rule_hits.json | 90 + .../tasks/index_kafka_security_rule_hits.json | 83 + ...ndex_kafka_service_chaining_rule_hits.json | 84 + .../index_kafka_service_function_status.json | 74 + .../tasks/index_kafka_statistics_rule.json | 367 ++ .../index_kafka_statistics_rule_hits.json | 159 + .../tasks/index_kafka_sys_storage_log.json | 52 + .../tasks/index_kafka_top_client_ips.json | 104 + .../tasks/index_kafka_top_external_ips.json | 105 + .../tasks/index_kafka_top_internal_ips.json | 105 + .../tasks/index_kafka_top_server_domains.json | 105 + .../tasks/index_kafka_top_server_fqdns.json | 105 + .../tasks/index_kafka_top_server_ips.json | 105 + .../tasks/index_kafka_top_subscribers.json | 105 + .../index_kafka_traffic_general_stat.json | 105 + ...index_kafka_traffic_shaping_rule_hits.json | 101 + ...raffic_top_destination_ip_metrics_log.json | 61 + .../roles/initialization/druid/vars/main.yml | 11 + .../galaxy-hos-service/tasks/init.yml | 28 + .../galaxy-hos-service/tasks/main.yml | 5 + .../galaxy-hos-service/tasks/upgrade.yml | 2 + .../galaxy-hos-service/vars/main.yml | 6 + .../galaxy-job-service/handlers/main.yml | 41 + .../galaxy-job-service/tasks/backup.yml | 51 + .../galaxy-job-service/tasks/check.yml | 12 + .../galaxy-job-service/tasks/deploy-admin.yml | 52 + .../tasks/deploy-executor.yml | 45 + .../galaxy-job-service/tasks/init.yml | 53 + .../galaxy-job-service/tasks/main.yml | 22 + .../tasks/uninstall-admin.yml | 18 + .../tasks/uninstall-executor.yml | 18 + .../galaxy-job-service/tasks/upgrade.yml | 54 + .../templates/admin/application.properties.j2 | 23 + .../admin/create_mariadb_tables.sh.j2 | 6 + .../templates/admin/docker-compose.yml.j2 | 15 + .../templates/admin/galaxy-job-admin.conf.j2 | 16 + .../admin/galaxy-job-admin.properties.j2 | 73 + .../templates/admin/logback.xml.j2 | 45 + .../templates/admin/xxl_job.sql.j2 | 195 + .../executor/application.properties.j2 | 23 + .../templates/executor/docker-compose.yml.j2 | 19 + .../galaxy-job-executor.properties.j2 | 65 + .../templates/executor/logback.xml.j2 | 43 + .../templates/push_config.sh.j2 | 52 + .../templates/upgrade-mariadb-data.sh.j2 | 50 + .../galaxy-job-service/vars/main.yml | 26 + .../galaxy-qgw-service/handlers/main.yml | 42 + .../galaxy-qgw-service/tasks/backup.yml | 25 + .../galaxy-qgw-service/tasks/check.yml | 24 + .../galaxy-qgw-service/tasks/deploy.yml | 47 + .../galaxy-qgw-service/tasks/init.yml | 60 + .../galaxy-qgw-service/tasks/main.yml | 17 + .../galaxy-qgw-service/tasks/uninstall.yml | 18 + .../galaxy-qgw-service/tasks/upgrade.yml | 95 + .../templates/application.yml.j2 | 22 + .../templates/create_mariadb_tables.sh.j2 | 4 + .../templates/docker-compose.yml.j2 | 14 + .../templates/galaxy-qgw-service.conf.j2 | 24 + .../templates/galaxy-qgw-service.yml.j2 | 175 + .../templates/hazelcast.yaml.j2 | 16 + .../galaxy-qgw-service/templates/keys.txt | 1 + .../templates/merge-config.sh.j2 | 83 + .../templates/push_config.sh.j2 | 52 + .../templates/upload_knowlodge.sh.j2 | 87 + .../galaxy-qgw-service/vars/main.yml | 16 + .../roles/initialization/hbase/tasks/init.yml | 54 + .../roles/initialization/hbase/tasks/main.yml | 5 + .../initialization/hbase/tasks/upgrade.yml | 41 + .../hbase/templates/create-hbase-table.sh | 15 + .../hbase/templates/create-phoenix-table.sh | 17 + .../initialization/hbase/templates/rsgroup.sh | 18 + .../hbase/templates/upgrade-hbase-data.sh | 48 + .../hbase/templates/upgrade-phoenix-data.sh | 48 + .../roles/initialization/kafka/tasks/init.yml | 19 + .../roles/initialization/kafka/tasks/main.yml | 2 + .../kafka/templates/create_topic.sh.j2 | 71 + .../roles/initialization/kafka/vars/main.yml | 1 + .../roles/initialization/nacos/tasks/main.yml | 14 + .../saved-query-scheduler/handlers/main.yml | 43 + .../saved-query-scheduler/tasks/backup.yml | 25 + .../saved-query-scheduler/tasks/check.yml | 23 + .../saved-query-scheduler/tasks/deploy.yml | 29 + .../saved-query-scheduler/tasks/init.yml | 25 + .../saved-query-scheduler/tasks/main.yml | 19 + .../saved-query-scheduler/tasks/uninstall.yml | 18 + .../saved-query-scheduler/tasks/upgrade.yml | 22 + .../templates/application.yml.j2 | 20 + .../templates/docker-compose.yml.j2 | 14 + .../templates/log4j2-dev.xml.j2 | 56 + .../templates/push_config.sh.j2 | 52 + .../templates/saved-query-scheduler.yml.j2 | 126 + .../saved-query-scheduler/vars/main.yml | 18 + .../schedule-service/tasks/init.yml | 51 + .../schedule-service/tasks/main.yml | 9 + .../schedule-service/tasks/uninstall.yml | 4 + .../dos-baseline/application.properties.j2 | 85 + .../iplearning/application.properties.j2 | 45 + .../templates/iplearning/start.sh.j2 | 43 + .../templates/mmdb/mmdb-perl.pl.j2 | 146 + .../schedule-service/vars/main.yml | 7 + .../initialization/yarn/tasks/backup.yml | 25 + .../initialization/yarn/tasks/bak.check.yml | 27 + .../roles/initialization/yarn/tasks/check.yml | 65 + .../roles/initialization/yarn/tasks/init.yml | 154 + .../roles/initialization/yarn/tasks/main.yml | 16 + .../initialization/yarn/tasks/uninstall.yml | 14 + .../config/agg_app_protocol_traffic.j2 | 27 + .../env/agg_app_protocol_traffic.sh | 14 + .../config/detection_dos_attack.j2 | 50 + .../dos-detection/env/detection_dos_attack.sh | 16 + .../config/agg_traffic_file_chunk_combine.j2 | 41 + .../agg_troubleshooting_file_chunk_combine.j2 | 41 + .../env/agg_traffic_file_chunk_combine.sh | 15 + .../agg_troubleshooting_file_chunk_combine.sh | 15 + .../config/dos_event_kafka_to_clickhouse | 53 + .../etl_proxy_event_kafka_to_clickhouse | 161 + .../etl_session_record_kafka_to_clickhouse | 161 + ...etl_transaction_record_kafka_to_clickhouse | 159 + .../etl_voip_record_kafka_to_clickhouse | 161 + .../env/dos_event_kafka_to_clickhouse.sh | 14 + .../etl_proxy_event_kafka_to_clickhouse.sh | 15 + .../etl_session_record_kafka_to_clickhouse.sh | 15 + ..._transaction_record_kafka_to_clickhouse.sh | 15 + .../etl_voip_record_kafka_to_clickhouse.sh | 15 + .../yarn/templates/groot-stream/start.sh | 64 + .../yarn/templates/groot-stream/stop.sh | 50 + .../config/correlation_sip_rtp_session.j2 | 46 + .../env/correlation_sip_rtp_session.sh | 15 + .../initialization/yarn/templates/start.sh | 68 + .../initialization/yarn/templates/stop.sh | 56 + .../config/agg_session_record_topn.j2 | 69 + .../env/agg_session_record_topn.sh | 13 + .../yarn/templates/upload_knowlodge.sh.j2 | 6 + .../roles/initialization/yarn/vars/main.yml | 21 + 170 files changed, 13137 insertions(+) create mode 100644 tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml create mode 100644 tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml create mode 100644 tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml create mode 100644 tsg-olap/parcels/roles/initialization/druid/tasks/check.yml create mode 100644 tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml create mode 100644 tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml create mode 100644 tsg-olap/parcels/roles/initialization/druid/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/s.bak create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json create mode 100644 tsg-olap/parcels/roles/initialization/druid/vars/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/handlers/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/handlers/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/backup.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/check.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/deploy.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/uninstall.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/upgrade.yml create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/application.yml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/create_mariadb_tables.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/docker-compose.yml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.conf.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/hazelcast.yaml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/keys.txt create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/merge-config.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/push_config.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/upload_knowlodge.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/galaxy-qgw-service/vars/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/hbase/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/hbase/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/hbase/tasks/upgrade.yml create mode 100644 tsg-olap/parcels/roles/initialization/hbase/templates/create-hbase-table.sh create mode 100644 tsg-olap/parcels/roles/initialization/hbase/templates/create-phoenix-table.sh create mode 100644 tsg-olap/parcels/roles/initialization/hbase/templates/rsgroup.sh create mode 100644 tsg-olap/parcels/roles/initialization/hbase/templates/upgrade-hbase-data.sh create mode 100644 tsg-olap/parcels/roles/initialization/hbase/templates/upgrade-phoenix-data.sh create mode 100644 tsg-olap/parcels/roles/initialization/kafka/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/kafka/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/kafka/templates/create_topic.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/kafka/vars/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/nacos/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/handlers/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/backup.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/check.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/deploy.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/uninstall.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/upgrade.yml create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/application.yml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/docker-compose.yml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/log4j2-dev.xml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/push_config.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/saved-query-scheduler.yml.j2 create mode 100644 tsg-olap/parcels/roles/initialization/saved-query-scheduler/vars/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/schedule-service/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/schedule-service/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/schedule-service/tasks/uninstall.yml create mode 100644 tsg-olap/parcels/roles/initialization/schedule-service/templates/dos-baseline/application.properties.j2 create mode 100644 tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/application.properties.j2 create mode 100644 tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/start.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/schedule-service/templates/mmdb/mmdb-perl.pl.j2 create mode 100644 tsg-olap/parcels/roles/initialization/schedule-service/vars/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/yarn/tasks/backup.yml create mode 100644 tsg-olap/parcels/roles/initialization/yarn/tasks/bak.check.yml create mode 100644 tsg-olap/parcels/roles/initialization/yarn/tasks/check.yml create mode 100644 tsg-olap/parcels/roles/initialization/yarn/tasks/init.yml create mode 100644 tsg-olap/parcels/roles/initialization/yarn/tasks/main.yml create mode 100644 tsg-olap/parcels/roles/initialization/yarn/tasks/uninstall.yml create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/config/agg_app_protocol_traffic.j2 create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/env/agg_app_protocol_traffic.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/config/detection_dos_attack.j2 create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/env/detection_dos_attack.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_traffic_file_chunk_combine.j2 create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_troubleshooting_file_chunk_combine.j2 create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_traffic_file_chunk_combine.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_troubleshooting_file_chunk_combine.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/dos_event_kafka_to_clickhouse create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_proxy_event_kafka_to_clickhouse create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_session_record_kafka_to_clickhouse create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_transaction_record_kafka_to_clickhouse create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_voip_record_kafka_to_clickhouse create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/dos_event_kafka_to_clickhouse.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_proxy_event_kafka_to_clickhouse.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_session_record_kafka_to_clickhouse.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_transaction_record_kafka_to_clickhouse.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_voip_record_kafka_to_clickhouse.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/start.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/stop.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/config/correlation_sip_rtp_session.j2 create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/env/correlation_sip_rtp_session.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/start.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/stop.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/config/agg_session_record_topn.j2 create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/env/agg_session_record_topn.sh create mode 100644 tsg-olap/parcels/roles/initialization/yarn/templates/upload_knowlodge.sh.j2 create mode 100644 tsg-olap/parcels/roles/initialization/yarn/vars/main.yml diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml new file mode 100644 index 0000000..dfb0b32 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml @@ -0,0 +1,11 @@ +- name: Set arangodb check variable + set_fact: + pin: "root:{{ arangodb_default_pin }}" + +- name: Check if collection exists in ArangoDB + uri: + url: "http://{{ groups.arangodb[0] }}:8529/_db/{{ arangodb_database }}/_api/collection/{{ item }}" + method: GET + headers: + Authorization: Basic {{ pin | b64encode }} + loop: "{{ arangodb_collections }}" diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml new file mode 100644 index 0000000..920b6d4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml @@ -0,0 +1,21 @@ +- name: Creating directory + file: + state: directory + path: '{{ deploy_dir }}/{{ container_name }}/init' + +- name: Copying init files + template: + src: "create_database.sh.j2" + dest: "{{ deploy_dir }}/{{ container_name }}/init/create_database.sh" + mode: 0755 + force: true + +- name: Creating database and collection + shell: 'cd {{ deploy_dir }}/{{ container_name }}/init && sh create_database.sh' + register: response + until: response.rc == 0 + retries: 3 + delay: 3 + failed_when: response.rc != 0 + run_once: true + diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml new file mode 100644 index 0000000..8894d83 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml @@ -0,0 +1,3 @@ +- include: init.yml + when: operation == 'install' +- include: check.yml diff --git a/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 b/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 new file mode 100644 index 0000000..29657da --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 @@ -0,0 +1,54 @@ +#!/bin/bash +source /etc/profile + +arangodb_host={{ groups.arangodb[0] | default('localhost') }} +username=root +pin="{{ arangodb_default_pin }}" +database="{{ arangodb_database }}" + +function create_database() { + # 检查数据库是否存在 + sleep 2 + check_database=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" http://$arangodb_host:8529/_db/$database/_api/database/current) + if [[ $check_database -eq 200 ]]; then + echo "Database $database already exists." + else + # 创建数据库 + local response_code=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" -X POST -H "accept: application/json" -d "{\"name\": \"$database\"}" http://$arangodb_host:8529/_api/database) + if [[ $response_code -eq 201 ]]; then + echo "Database $database created successfully." + else + echo "Failed to create database $database." + exit 1 + fi + fi + sleep 2 + +} + +function create_collection() { + local collection_name=$1 + local collection_type=$2 + sleep 2 + # 检查文档是否存在 + check_collection=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" http://$arangodb_host:8529/_db/$database/_api/collection/$collection_name) + if [[ $check_collection -eq 200 ]]; then + echo "Collection $collection_name already exists." + else + # 创建文档$collection_name + local response_code=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" -X POST -H 'accept: application/json' -d "{\"name\": \"$collection_name\",\"type\": $collection_type }" http://$arangodb_host:8529/_db/$database/_api/collection?isSystem=false) + if [[ $response_code -eq 201 ]]; then + echo "Collection $collection_name created successfully." + else + echo "Failed to create collection $collection_name." + exit 1 + fi + fi + sleep 2 +} + +# 2: document 3: edges +create_database +create_collection IP 2 +create_collection R_LOCATE_FQDN2IP 3 + diff --git a/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml b/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml new file mode 100644 index 0000000..fa8b9d4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml @@ -0,0 +1,9 @@ +container_name: arangodb + +# 数据库 +arangodb_database: 'tsg_galaxy_v3' + +# 库表 +arangodb_collections: + - IP + - R_LOCATE_FQDN2IP diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql new file mode 100644 index 0000000..7eb3923 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql @@ -0,0 +1,4174 @@ +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query; + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,destination_ip,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_query ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,session_id,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.security_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app_transition String, + app LowCardinality(String), + app_debug_info String, + app_content String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + tunnels String, + dup_traffic_flag Nullable(Int32), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_port, + server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + app_debug_info, + app_content, + fqdn_category_list, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + tunnels, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(security_rule_list) = 0 +; + +-- tsg_galaxy_v3.monitor_event_materialized_view +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.monitor_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app_transition String, + app LowCardinality(String), + app_debug_info String, + app_content String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + tunnels String, + dup_traffic_flag Nullable(Int32), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_port, + server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + app_debug_info, + app_content, + fqdn_category_list, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + tunnels, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(monitor_rule_list) = 0 +; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql new file mode 100644 index 0000000..1f45639 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql @@ -0,0 +1,9 @@ +create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); +create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); +create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); +create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); +CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); +CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); + +alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; +alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql new file mode 100644 index 0000000..5e7ba07 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql @@ -0,0 +1,247 @@ +set distributed_ddl_task_timeout = 180; + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_domain,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + + +ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; + +ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; + +ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; + + +ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; + +ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; + +ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; + + +ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + +ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + +ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + + +drop view IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster; +drop view IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster; +drop view IF EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster; + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; + +drop table IF EXISTS `system`.query_log_cluster on cluster ck_query; +drop table IF EXISTS `system`.disks_cluster on cluster ck_query; +drop table IF EXISTS `system`.columns_cluster on cluster ck_query; +drop table IF EXISTS `system`.parts_cluster on cluster ck_query; +drop table IF EXISTS `system`.processes_cluster on cluster ck_query; +drop table IF EXISTS `system`.tables_cluster on cluster ck_query; + + +create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); +create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); +create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); +create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); +CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); +CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); +alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; +alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY; + diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql new file mode 100644 index 0000000..042b51f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql @@ -0,0 +1,110 @@ +set distributed_ddl_task_timeout = 180; + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS http_request_body_key String after common_flags; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS http_request_body_key String after common_flags; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS http_request_body_key String after common_flags; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + + + + + + + + + + + + + + + + diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql new file mode 100644 index 0000000..1d3d958 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql @@ -0,0 +1,47 @@ +set distributed_ddl_task_timeout = 180; + +alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; + + +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + + +alter table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + + +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + + + + + diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql new file mode 100644 index 0000000..0e50ac4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql @@ -0,0 +1,6 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.dos_event_local on cluster ck_cluster add column IF NOT EXISTS profile_id UInt64 after log_id; +ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster add column IF NOT EXISTS profile_id UInt64 after log_id; +ALTER table tsg_galaxy_v3.dos_event on cluster ck_query add column IF NOT EXISTS profile_id UInt64 after log_id; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql new file mode 100644 index 0000000..769c47e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql @@ -0,0 +1,11 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; + + +ALTER TABLE tsg_galaxy_v3.proxy_event_local on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; +ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; +ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_query DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql new file mode 100644 index 0000000..11c06a8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql @@ -0,0 +1,159 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; + + +ALTER TABLE tsg_galaxy_v3.proxy_event_local on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; +ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; +ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_query DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql new file mode 100644 index 0000000..27bf30b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql @@ -0,0 +1,97 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql new file mode 100644 index 0000000..df1ba02 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql @@ -0,0 +1,158 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.active_defence_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.active_defence_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql new file mode 100644 index 0000000..1528e0c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql @@ -0,0 +1,131 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql new file mode 100644 index 0000000..20b7e1e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql @@ -0,0 +1,76 @@ +set distributed_ddl_task_timeout = 180; + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml new file mode 100644 index 0000000..9c4c8b9 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml @@ -0,0 +1,24 @@ +- name: Setting clickhouse init directory + set_fact: + init_path: '{{ deploy_dir }}/clickhouse/init/' + +- name: Creatting clickhouse init directory + file: + path: '{{ init_path }}' + state: directory + +- name: Copying Clickhouse create tables sql + copy: + src: 'files/{{ item }}' + dest: '{{ init_path }}/{{ item }}' + force: true + with_items: + - create_ck_table.sql + - system.sql + +- name: Creating tables + shell: clickhouse-client -h {{ inventory_hostname }} --port 9001 -m -u default --password {{ clickhouse_default_pin }} --multiquery < {{ init_path }}/{{ item }} + loop: + - create_ck_table.sql + - system.sql + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml new file mode 100644 index 0000000..951be61 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml @@ -0,0 +1,5 @@ +- include: init.yml + when: inventory_hostname in groups['clickhouse'][:2] and operation == "install" + +- include: upgrate.yml + when: inventory_hostname in groups['clickhouse'][:2] and operation == "upgrade" diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml new file mode 100644 index 0000000..48401ee --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml @@ -0,0 +1,25 @@ +- name: Setting clickhouse upgrade directory + set_fact: + upgrade_path: '{{ deploy_dir }}/clickhouse/upgrade/' + +- name: Creatting clickhouse upgrade directory + file: + path: '{{ upgrade_path }}' + state: directory + +- name: Copying clickhouse upgrade tables sql + copy: + src: 'files/upgrade-sql' + dest: '{{ upgrade_path }}/' + force: true + +- name: Copying clickhouse upgrade script + template: + src: 'upgrade-clickhouse-data.sh.j2' + dest: '{{ upgrade_path }}/upgrade-clickhouse-data.sh' + mode: 0075 + force: true + +- name: Upgrade clickhouse data + shell: cd {{ upgrade_path }} && sh upgrade-clickhouse-data.sh + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 b/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 new file mode 100644 index 0000000..bc9fd19 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash +source /etc/profile +set -e + +# 数据库连接参数 +upgrade_dir=${1:-$(pwd)/upgrade-sql} + +# 指定当前版本和历史版本 +current_version="{{ solution_version }}" +old_version="{{ old_version }}" + +# 数据库连接参数 +db_hostname="127.0.0.1" +db_username="default" +db_password="galaxy2019" + +suffix=".sql" +# 获取更新目录中的所有SQL文件 +sql_files=$(find "${upgrade_dir}" -name "*$suffix") + +# 标记是否开始执行历史版本的标识 +start_execution=false + +# 循环处理每个SQL文件 +for file in ${sql_files}; do + # 从文件名中提取版本号 + filename=$(basename "$file") + version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.') + current_version="${current_version//./}" + old_version="${old_version//./}" + # 检查是否达到历史版本 + if [[ "${version}" -gt "${old_version}" ]]; then + start_execution=true + fi + + # 检查是否开始执行历史版本之后的SQL文件 + if [[ ${start_execution} = true ]]; then + echo "Executing SQL file: ${file}" + + # 执行SQL文件 + clickhouse-client -h 127.0.0.1 --port 9001 -m -u "${db_username}" --password "${db_password}" --multiquery <"${file}" + echo "Upgrade ${version} successfully" + fi + + # 检查是否达到当前版本 + if [[ "${version}" -ge "${current_version}" ]]; then + break + fi +done + diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml new file mode 100644 index 0000000..899461b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml @@ -0,0 +1,25 @@ +- name: Check if backup directory exists + stat: + path: "{{ backup_path }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/{{ container_name }}/{{ item }}" + dest: "{{ backup_path }}" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}" + state: absent + when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml new file mode 100644 index 0000000..295e895 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml @@ -0,0 +1,23 @@ +- name: Get druid running tasks + shell: curl -s http://{{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/indexer/v1/supervisor?state=true | jq -r '.[] | select(.state == "RUNNING") | .id' + register: run_task_list + run_once: true + +- name: Get init task template + find: + paths: "templates/tasks/" + file_type: file + patterns: "*.json" + register: find_result + delegate_to: 127.0.0.1 + run_once: true + +- name: Set init task list + set_fact: + init_task_list: "{{ find_result.files | map(attribute='path') | map('basename') | map('regex_replace', '^index_kafka_(.*)\\.json$', '\\1') | list }}" + +- name: Verify that the number of running tasks is correct + fail: + msg: "The number of running druid tasks is less than the normal number!" + when: run_task_list.stdout_lines | intersect(init_task_list) |length != (init_task_list|length) + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml new file mode 100644 index 0000000..aaff071 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml @@ -0,0 +1,115 @@ +- name: Setting init directory + set_fact: + init_path: '{{ deploy_dir }}/druid/init/' + topology_path: '{{ deploy_dir }}/druid/topology/' + +- name: Remove files and directories + file: + path: "{{ init_path }}" + state: absent + ignore_errors: true + loop: + - "{{ init_path }}" + - "{{ topology_path }}" + +- name: Creatting init directory + file: + path: '{{ item }}' + state: directory + loop: + - "{{ init_path }}" + - "{{ topology_path }}" + - "{{ topology_path }}/tasks" + +- name: Setting hdfs_ip variable + set_fact: + hdfs_ip: "{{groups.hdfs[0]}}" + +- name: Unzipping segments.zip + unarchive: + src: 'files/cluster/segments.zip' + dest: '{{ deploy_dir }}/' + force: true + run_once: true + delegate_to: '{{ hdfs_ip }}' + +- name: Copying push segments to hdfs installation path + copy: + src: 'files/cluster/push_segments.sh' + dest: '{{ deploy_dir }}/push_segments.sh' + force: true + mode: 0755 + run_once: true + delegate_to: '{{ hdfs_ip }}' + +- name: Creating druid directory and putting segments to hdfs + shell: source /etc/profile && hadoop fs -mkdir -p /druid/segments + register: nums_out + run_once: true + delegate_to: '{{ hdfs_ip }}' + ignore_errors: yes + +- name: Putting segments to hdfs + shell: source /etc/profile && cd {{ deploy_dir }}/ && sh push_segments.sh + register: nums_out + run_once: true + delegate_to: '{{ hdfs_ip }}' + +- name: Copying mysql Client to /usr/bin/ + copy: + src: 'files/mysql' + dest: '/usr/bin/' + mode: 0755 + force: true + +- name: Copying init files to {{ init_path }} + copy: + src: 'files/cluster/' + dest: '{{ deploy_dir }}/druid/init' + force: true + +- name: Copying tasks template to {{ topology_path }} + template: + src: "{{ item }}" + dest: "{{ deploy_dir }}/druid/topology/tasks/{{ item | basename | regex_replace('.j2$', '') }}" + mode: "0644" + force: true + with_fileglob: "templates/tasks/*" + +- name: Inserting segments to mariadb + shell: 'mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} druid < {{ init_path }}/druid_segments-tsg.sql' + run_once: true + +- name: Copying script files + template: + src: '{{ item }}.j2' + dest: '{{ topology_path }}/{{ item }}' + mode: 0755 + with_items: + - 'druid_monitor.sh' + - 'supervisor-manager.sh' + +- name: Copying monitor template + template: + src: 'run_druid_monitor.j2' + dest: '/etc/cron.d/run_druid_monitor' + mode: 0644 + +- name: Submitting druid tasks + shell: cd {{ topology_path }} && sh supervisor-manager.sh startall + run_once: true + +- name: Deleting {{ deploy_dir }}/push_segments.sh + file: + path: "{{ deploy_dir }}/push_segments.sh" + state: absent + run_once: true + delegate_to: '{{ hdfs_ip }}' + +- name: Deleting {{ deploy_dir }}/segments.zip + file: + path: "{{ deploy_dir }}/segments.zip" + state: absent + run_once: true + delegate_to: '{{ hdfs_ip }}' + diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml new file mode 100644 index 0000000..9c10b77 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml @@ -0,0 +1,65 @@ +- name: Setting init directory + set_fact: + init_path: '{{ deploy_dir }}/druid/init/' + topology_path: '{{ deploy_dir }}/druid/topology/' + +- name: Creatting init directory + file: + path: '{{ item }}' + state: directory + loop: + - '{{ init_path }}' + - '{{ topology_path }}' + - '{{ topology_path }}/tasks' + - '{{ deploy_dir }}/druid/var/druid' + +- name: Unzipping segments.zip + unarchive: + src: 'files/standalone/segments.zip' + dest: '{{ deploy_dir }}/druid/var/druid/' + force: true + +- name: Copying mysql Client to /usr/bin/ + copy: + src: 'files/mysql' + dest: '/usr/bin/' + mode: 0755 + force: true + +- name: Copying init files to {{ init_path }} + copy: + src: 'files/standalone' + dest: '{{ init_path }}/' + force: true + +- name: Copying tasks template to {{ topology_path }} + template: + src: "{{ item }}" + dest: "{{ deploy_dir }}/druid/topology/tasks/{{ item | basename | regex_replace('.j2$', '') }}" + mode: "0644" + force: true + with_fileglob: "templates/tasks/*" + + +- name: Inserting segments to mariadb + shell: 'mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} druid < {{ init_path }}/standalone/druid_segments-tsg.sql' + run_once: true + +- name: Copying script files + template: + src: '{{ item }}.j2' + dest: '{{ topology_path }}/{{ item }}' + mode: 0755 + with_items: + - "druid_monitor.sh" + - "supervisor-manager.sh" + +- name: Copying monitor template + template: + src: 'run_druid_monitor.j2' + dest: '/etc/cron.d/run_druid_monitor' + mode: 0644 + +- name: Submitting druid tasks + shell: cd {{ topology_path }} && sh supervisor-manager.sh startall + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml new file mode 100644 index 0000000..35bf1ff --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml @@ -0,0 +1,21 @@ +- block: + - include: uninstall.yml + - include: "{{ playbook_name }}" + vars: + playbook_name: "{{ 'init-cluster.yml' if groups.druid | length > 1 else 'init-standalone.yml' }}" + - include: check.yml + when: inventory_hostname in groups['druid'][:2] and operation == 'install' + +- block: + - include: backup.yml + - include: uninstall.yml + - include: "{{ playbook_name }}" + vars: + playbook_name: "{{ 'init-cluster.yml' if groups.druid | length > 1 else 'init-standalone.yml' }}" + - include: check.yml + when: inventory_hostname in groups['druid'][:2] and (operation) == "upgrade" + +- block: + - include: uninstall.yml + when: inventory_hostname in groups['druid'][:2] and (operation) == "uninstall" + diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml new file mode 100644 index 0000000..ec46d7c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml @@ -0,0 +1,30 @@ +- name: Check {{ deploy_dir }}/druid/topology + stat: + path: "{{ deploy_dir }}/druid/topology" + register: directory_stat + +- block: + - name: Copying supervisor-manager.sh template + template: + src: 'supervisor-manager.sh.j2' + dest: '{{ deploy_dir }}/druid/topology/supervisor-manager.sh' + mode: 0755 + + - name: Stop all tasks + shell: cd {{ deploy_dir }}/druid/topology && sh supervisor-manager.sh terminateall + run_once: true + + - name: Get tasks status + shell: curl -s http://{{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/indexer/v1/supervisor?state=true | jq 'map(select(.state = "RUNNING" ))' | jq 'length' + register: tasks_count + + - name: Check tasks status + debug: + msg: "{{ tasks_count.stdout }} tasks are not stopped" + failed_when: tasks_count.stdout > '0' + + - name: Delete {{ deploy_dir }}/druid/topology + file: + path: "{{ deploy_dir }}/druid/topology" + state: absent + when: directory_stat.stat.exists diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 new file mode 100644 index 0000000..e0c0444 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 @@ -0,0 +1,58 @@ +#!/bin/bash + +druid_ip={{ vrrp_instance.default.virtual_ipaddress }} +druid_port=8089 +druid_monitor_prom_file={{ deploy_dir }}/node-exporter/prom/druid_monitor.prom + +task_name=`cat tasklist` + +#获取supervisor状态信息 +function getSupervisorStatus(){ +druid_supervisor_healthy_num=`curl -G -d 'state=true' -s $druid_ip:$druid_port/druid/indexer/v1/supervisor | jq '[ .[] | .state] | length'` +echo druid_supervisor_healthy_num $druid_supervisor_healthy_num >> $druid_monitor_prom_file + +druid_supervisor_unhealthy_num=`curl -G -d 'state=true' -s $druid_ip:$druid_port/druid/indexer/v1/supervisor | jq '[ .[] | select(.state != "RUNNING")] | length'` +echo druid_supervisor_unhealthy_num $druid_supervisor_unhealthy_num >> $druid_monitor_prom_file +} + +#获取Druid 任务状态信息 +function getTaskStatus(){ +druid_index_running_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/runningTasks | jq '. | length'` +echo druid_index_running_task_num $druid_index_running_task_num > $druid_monitor_prom_file + +druid_index_waiting_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/waitingTasks | jq '. | length'` +echo druid_index_waiting_task_num $druid_index_waiting_task_num >> $druid_monitor_prom_file + +druid_index_pending_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/pendingTasks | jq '. | length'` +echo druid_index_pending_task_num $druid_index_pending_task_num >> $druid_monitor_prom_file + +druid_compact_pending_task_num=`curl -G -d 'type=compact' -s $druid_ip:$druid_port/druid/indexer/v1/pendingTasks | jq '. | length'` +echo druid_compact_pending_task_num $druid_compact_pending_task_num >> $druid_monitor_prom_file + +druid_compact_waiting_task_num=`curl -G -d 'type=compact' -s $druid_ip:$druid_port/druid/indexer/v1/waitingTasks | jq '. | length'` +echo druid_compact_waiting_task_num $druid_compact_waiting_task_num >> $druid_monitor_prom_file +} + +#校验正在运行的index数量，与启动时指定的index数量对比。 +#>=指定数值判断为正常 返回：0 +#反之异常 返回：1 +function checkIndexStatus(){ +druid_task_sum=0 +for var in ${task_name[@]}; +do + druid_task_num=`cat tasks/${var} | jq .ioConfig.taskCount` + druid_task_sum=`expr $druid_task_num + $druid_task_sum` +done + +druid_index_task_healthy_flag=0 +if [ $druid_index_running_task_num -lt $druid_task_sum ] +then + druid_index_task_healthy_flag=1 +fi + +echo druid_index_task_healthy_flag $druid_index_task_healthy_flag >> $druid_monitor_prom_file +} + +getTaskStatus +getSupervisorStatus +checkIndexStatus diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 new file mode 100644 index 0000000..bbffdc4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/bash + +base_dir=`dirname $0` +echo $base_dir + +#router节点IP +router_ip={{ groups.druid[0] }} + +common_task_name=`cat $base_dir/common_task.txt` +common_rule_file="$base_dir/common_data_rule.json" + +hot_task_name=`cat $base_dir/hot_task.txt` +hot_rule_file="$base_dir/hot_data_rule.json" + +for var in ${common_task_name[@]}; +do +curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @$common_rule_file http://${router_ip}:8088/druid/coordinator/v1/rules/${var} +echo "'${var}' 任务启动成功" +sleep 2 +done + +for var in ${hot_task_name[@]}; +do +curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @$hot_rule_file http://${router_ip}:8088/druid/coordinator/v1/rules/${var} +echo "'${var}' 任务启动成功" +sleep 2 +done + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 new file mode 100644 index 0000000..88da982 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 @@ -0,0 +1,75 @@ +#!/bin/bash + +base_dir=`dirname $0` +base_dir="$(cd "$base_dir" && pwd)" + +router_ip='{{ vrrp_instance.default.virtual_ipaddress }}' + +common_task_name=`cat $base_dir/tasklist` +common_rule_file="$base_dir/common_data_rule.json" + +hot_task_name=`cat $base_dir/hot_task.txt` +hot_rule_file="$base_dir/hot_data_rule.json" + +case $1 in + + resetAll) + for var in ${common_task_name[@]}; + do + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset + sleep 2 + done + + for var in ${hot_task_name[@]}; + do + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset + sleep 2 + done + ;; + + terminateAll) + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/terminateAll + ;; + + shutdownAllTasks) + for var in ${common_task_name[@]}; + do + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks + sleep 2 + done + + for var in ${hot_task_name[@]}; + do + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks + sleep 2 + done + ;; + + reset) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger reset " + exit 1 + fi + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/reset + ;; + + terminate) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger terminate " + exit 1 + fi + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/terminate + ;; + + shutdownTasks) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger shutdownTasks " + exit 1 + fi + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/$2/shutdownAllTasks + ;; + *) + echo "Usage: supervisor-manger {resetAll|terminateAll|shutdownAllTasks}" + echo "Usage: supervisor-manger {reset|terminate|shutdownTasks} " + ;; +esac diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 new file mode 100644 index 0000000..d5fee98 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 @@ -0,0 +1,3 @@ +#Ansible: Check druid up to node_exporter +*/10 * * * * root cd {{ topology_path }} && sh druid_monitor.sh + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/s.bak b/tsg-olap/parcels/roles/initialization/druid/templates/s.bak new file mode 100644 index 0000000..ed547af --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/s.bak @@ -0,0 +1,124 @@ +#!/bin/bash +source /etc/profile + +DRUID_HOST='{{ vrrp_instance.default.virtual_ipaddress }}' +KAFKA_HOST="{% for host in groups['kafka'] %} +{{ host }}:9094{% if not loop.last %},{% endif %} +{% endfor %}" + +# 参数：1.操作 2.任务文件 +OPERATION=$1 + +TASK_LIST=( + index_kafka_application_protocol_stat.json + index_kafka_monitor_rule_hits.json + index_kafka_object_statistics.json + index_kafka_proxy_rule_hits.json + index_kafka_security_rule_hits.json + index_kafka_service_chaining_rule_hits.json + index_kafka_service_function_status.json + index_kafka_statistics_rule_hits.json + index_kafka_statistics_rule.json + index_kafka_sys_storage_log.json + index_kafka_top_client_ips.json + index_kafka_top_external_ips.json + index_kafka_top_internal_ips.json + index_kafka_top_server_domains.json + index_kafka_top_server_fqdns.json + index_kafka_top_server_ips.json + index_kafka_top_subscribers.json + index_kafka_traffic_general_stat.json + index_kafka_traffic_shaping_rule_hits.json + index_kafka_traffic_top_destination_ip_metrics_log.json +) + +if [ $# -lt 1 ]; then + echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" + echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] " + exit 1 +fi + +function start_task() { + local task_name=$1 + cp ./tasks/${task_name} ./ + + sed -i 's/kafkabootstrap/'$KAFKA_HOST'/' ${task_name} + + curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @${task_name} http://$DRUID_HOST:8089/druid/indexer/v1/supervisor + + if [ $? -eq "0" ]; then + echo "task:'${task_name}' Start-up success! " + rm -rf ./${task_name} + else + echo "task:'${task_name}' Start-up failure! " + exit 1 + fi + +} + +function reset_task() { + local task_name=$1 + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/reset" + sleep 2 +} + +function terminate_task() { + local task_name=$1 + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/terminate" +} + +function terminate_all() { + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/terminateAll" +} + +function shutdown_tasks() { + local task_name=$1 + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + local task_name=$1 + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/datasources/${task_name}/shutdownAllTasks" + sleep 2 +} + +case $OPERATION in +start) + start_task "$2" + ;; +startall) + for task in ${TASK_LIST[@]}; do + start_task "$task" + done + ;; +reset) + reset_task "$2" + ;; +resetall) + for task in ${TASK_LIST[@]}; do + reset_task "$task" + done + ;; +terminate) + terminate_task "$2" + ;; +terminateall) + terminate_all + ;; +shutdown) + shutdown_tasks "$2" + ;; +shutdownall) + for task in ${TASK_LIST[@]}; do + shutdown_tasks "$task" + done + ;; + +*) + echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" + echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] " + ;; +esac + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 new file mode 100644 index 0000000..e140a07 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 @@ -0,0 +1,103 @@ +#!/bin/bash +source /etc/profile + +DRUID_HOST='{{ vrrp_instance.default.virtual_ipaddress }}' +KAFKA_HOST="{% for host in groups['kafka'] %} +{{ host }}:9094{% if not loop.last %},{% endif %} +{% endfor %}" + +# 参数：1.操作 2.任务文件或目录 +OPERATION=$1 +filename=$(basename "$file") +TASK_FILE_LIST=$(find "./tasks" -name "*.json") + +if [ $# -lt 1 ]; then + echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" + echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <./tasks/[task file]>" + exit 1 +fi + +function start_task() { + local task_file=$1 + local task_name=$(basename $task_file) + cp ${task_file} ./ + + sed -i 's/kafkabootstrap/'$KAFKA_HOST'/' ${task_name} + + curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @${task_name} http://$DRUID_HOST:8089/druid/indexer/v1/supervisor + + if [ $? -eq "0" ]; then + echo "task:'${task_name}' Start-up success! " + rm -rf ./${task_name} + else + echo "task:'${task_name}' Start-up failure! " + exit 1 + fi + +} + +function reset_task() { + local task_name=$(basename $1) + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/reset" + sleep 2 +} + +function terminate_task() { + local task_name=$(basename $1) + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/terminate" +} + +function terminate_all() { + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/terminateAll" +} + +function shutdown_tasks() { + local task_name=$(basename $1) + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/datasources/${task_name}/shutdownAllTasks" + sleep 2 +} + +case $OPERATION in +start) + start_task "$2" + ;; +startall) + for task in ${TASK_FILE_LIST[@]}; do + start_task "$task" + done + ;; +reset) + reset_task "$2" + ;; +resetall) + for task in ${TASK_FILE_LIST[@]}; do + reset_task "$task" + done + ;; +terminate) + terminate_task "$2" + ;; +terminateall) + terminate_all + ;; +shutdown) + shutdown_tasks "$2" + ;; +shutdownall) + for task in ${TASK_FILE_LIST[@]}; do + shutdown_tasks "$task" + done + ;; + +*) + echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" + echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <./tasks/[task file]>" + ;; +esac + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json new file mode 100644 index 0000000..96f992a --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json @@ -0,0 +1,131 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "application_protocol_stat", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + "protocol_stack_id", + "app_name", + {"name": "vsys_id", "type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + {"name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id"}, + {"name": "data_center", "type": "path", "expr": "$.tags.data_center"}, + {"name": "device_group", "type": "path", "expr": "$.tags.device_group"}, + {"name": "device_id", "type": "path", "expr": "$.tags.device_id"}, + {"name": "protocol_stack_id", "type": "path", "expr": "$.tags.protocol_stack_id"}, + {"name": "app_name", "type": "path", "expr": "$.tags.app_name"}, + {"name": "sessions", "type": "path", "expr": "$.fields.sessions"}, + {"name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes"}, + {"name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes"}, + {"name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts"}, + {"name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts"}, + {"name": "c2s_pkts", "type": "path", "expr": "$.fields.c2s_pkts"}, + {"name": "s2c_pkts", "type": "path", "expr": "$.fields.s2c_pkts"}, + {"name": "c2s_bytes", "type": "path", "expr": "$.fields.c2s_bytes"}, + {"name": "s2c_bytes", "type": "path", "expr": "$.fields.s2c_bytes"}, + {"name": "c2s_fragments", "type": "path", "expr": "$.fields.c2s_fragments"}, + {"name": "s2c_fragments", "type": "path", "expr": "$.fields.s2c_fragments"}, + {"name": "c2s_tcp_lost_bytes", "type": "path", "expr": "$.fields.c2s_tcp_lost_bytes"}, + {"name": "s2c_tcp_lost_bytes", "type": "path", "expr": "$.fields.s2c_tcp_lost_bytes"}, + {"name": "c2s_tcp_ooorder_pkts", "type": "path", "expr": "$.fields.c2s_tcp_ooorder_pkts"}, + {"name": "s2c_tcp_ooorder_pkts", "type": "path", "expr": "$.fields.s2c_tcp_ooorder_pkts"}, + {"name": "c2s_tcp_retransmitted_pkts", "type": "path", "expr": "$.fields.c2s_tcp_retransmitted_pkts"}, + {"name": "s2c_tcp_retransmitted_pkts", "type": "path", "expr": "$.fields.s2c_tcp_retransmitted_pkts"}, + {"name": "c2s_tcp_retransmitted_bytes", "type": "path", "expr": "$.fields.c2s_tcp_retransmitted_bytes"}, + {"name": "s2c_tcp_retransmitted_bytes", "type": "path", "expr": "$.fields.s2c_tcp_retransmitted_bytes"} + ] + } + } + }, + "metricsSpec": [ + {"type": "longSum", "name": "sessions", "fieldName": "sessions"}, + {"type": "longSum", "name": "c2s_pkts", "fieldName": "c2s_pkts"}, + {"type": "longSum", "name": "s2c_pkts", "fieldName": "s2c_pkts"}, + {"type": "longSum", "name": "c2s_bytes", "fieldName": "c2s_bytes"}, + {"type": "longSum", "name": "s2c_bytes", "fieldName": "s2c_bytes"}, + {"type": "longSum", "name": "c2s_fragments", "fieldName": "c2s_fragments"}, + {"type": "longSum", "name": "s2c_fragments", "fieldName": "s2c_fragments"}, + {"type": "longSum", "name": "c2s_tcp_lost_bytes", "fieldName": "c2s_tcp_lost_bytes"}, + {"type": "longSum", "name": "s2c_tcp_lost_bytes", "fieldName": "s2c_tcp_lost_bytes"}, + {"type": "longSum", "name": "c2s_tcp_ooorder_pkts", "fieldName": "c2s_tcp_ooorder_pkts"}, + {"type": "longSum", "name": "s2c_tcp_ooorder_pkts", "fieldName": "s2c_tcp_ooorder_pkts"}, + {"type": "longSum", "name": "c2s_tcp_retransmitted_pkts", "fieldName": "c2s_tcp_retransmitted_pkts"}, + {"type": "longSum", "name": "s2c_tcp_retransmitted_pkts", "fieldName": "s2c_tcp_retransmitted_pkts"}, + {"type": "longSum", "name": "c2s_tcp_retransmitted_bytes", "fieldName": "c2s_tcp_retransmitted_bytes"}, + {"type": "longSum", "name": "s2c_tcp_retransmitted_bytes", "fieldName": "s2c_tcp_retransmitted_bytes"}, + {"type": "longSum", "name": "in_bytes", "fieldName": "in_bytes"}, + {"type": "longSum", "name": "out_bytes", "fieldName": "out_bytes"}, + {"type": "longSum", "name": "in_pkts", "fieldName": "in_pkts"}, + {"type": "longSum", "name": "out_pkts", "fieldName": "out_pkts"} + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S"}, + "rollup": true + }, + "transformSpec": { + "transforms": [ + {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "c2s_pkts", "expression": "nvl(c2s_pkts, 0)"}, + {"type": "expression", "name": "s2c_pkts", "expression": "nvl(s2c_pkts, 0)"}, + {"type": "expression", "name": "c2s_bytes", "expression": "nvl(c2s_bytes, 0)"}, + {"type": "expression", "name": "s2c_bytes", "expression": "nvl(s2c_bytes, 0)"}, + {"type": "expression", "name": "c2s_fragments", "expression": "nvl(c2s_fragments, 0)"}, + {"type": "expression", "name": "s2c_fragments", "expression": "nvl(s2c_fragments, 0)"}, + {"type": "expression", "name": "c2s_tcp_lost_bytes", "expression": "nvl(c2s_tcp_lost_bytes, 0)"}, + {"type": "expression", "name": "s2c_tcp_lost_bytes", "expression": "nvl(s2c_tcp_lost_bytes, 0)"}, + {"type": "expression", "name": "c2s_tcp_ooorder_pkts", "expression": "nvl(c2s_tcp_ooorder_pkts, 0)"}, + {"type": "expression", "name": "s2c_tcp_ooorder_pkts", "expression": "nvl(s2c_tcp_ooorder_pkts, 0)"}, + {"type": "expression", "name": "c2s_tcp_retransmitted_pkts", "expression": "nvl(c2s_tcp_retransmitted_pkts, 0)"}, + {"type": "expression", "name": "s2c_tcp_retransmitted_pkts", "expression": "nvl(s2c_tcp_retransmitted_pkts, 0)"}, + {"type": "expression", "name": "c2s_tcp_retransmitted_bytes", "expression": "nvl(c2s_tcp_retransmitted_bytes, 0)"}, + {"type": "expression", "name": "s2c_tcp_retransmitted_bytes", "expression": "nvl(s2c_tcp_retransmitted_bytes, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { + "type": "selector", + "dimension": "name", + "value": "application_protocol_stat" + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "NETWORK-TRAFFIC-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json new file mode 100644 index 0000000..5c43f46 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json @@ -0,0 +1,83 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "monitor_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + {"name": "rule_id","type": "long"}, + {"name": "action","type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "action", "type": "path", "expr": "$.tags.action" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "monitor_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json new file mode 100644 index 0000000..84dd11b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json @@ -0,0 +1,95 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "object_statistics", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + {"name": "template_id","type": "long"}, + {"name": "chart_id","type": "long"}, + {"name": "version","type": "long"}, + "object_type", + {"name": "object_id","type": "long"}, + {"name": "item_id","type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "bytes", "type": "path", "expr": "$.fields.bytes" }, + { "name": "new_in_sessions", "type": "path", "expr": "$.fields.new_in_sessions" }, + { "name": "new_out_sessions", "type": "path", "expr": "$.fields.new_out_sessions" }, + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "template_id", "type": "path", "expr": "$.tags.template_id" }, + { "name": "chart_id", "type": "path", "expr": "$.tags.chart_id" }, + { "name": "version", "type": "path", "expr": "$.tags.version" }, + { "name": "object_type", "type": "path", "expr": "$.tags.object_type" }, + { "name": "object_id", "type": "path", "expr": "$.tags.object_id" }, + { "name": "item_id", "type": "path", "expr": "$.tags.item_id" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" }, + { "type" : "longSum", "name" : "new_in_sessions", "fieldName" : "new_in_sessions" }, + { "type" : "longSum", "name" : "new_out_sessions", "fieldName" : "new_out_sessions" }, + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"}, + {"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"}, + {"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "object_statistics" } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "OBJECT-STATISTICS-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json new file mode 100644 index 0000000..ab97fb1 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json @@ -0,0 +1,90 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "proxy_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { "name": "vsys_id", "type": "long" }, + { "name": "rule_id", "type": "long" }, + { "name": "pinning_status", "type": "string" }, + { "name": "action", "type": "long" }, + { "name": "sub_action", "type": "string" } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "pinning_status", "type": "path", "expr": "$.tags.pinning_status" }, + { "name": "action", "type": "path", "expr": "$.tags.action" }, + { "name": "sub_action", "type": "path", "expr": "$.tags.sub_action" }, + { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "proxy_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json new file mode 100644 index 0000000..957923d --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json @@ -0,0 +1,83 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "security_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + {"name": "rule_id","type": "long"}, + {"name": "action","type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "action", "type": "path", "expr": "$.tags.action" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "security_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json new file mode 100644 index 0000000..6f3e3bd --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json @@ -0,0 +1,84 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "service_chaining_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { "name": "vsys_id", "type": "long" }, + { "name": "rule_id", "type": "long" }, + { "name": "sff_profile_id", "type": "long" }, + { "name": "sf_profile_id", "type": "long" } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "sff_profile_id", "type": "path", "expr": "$.tags.sff_profile_id" }, + { "name": "sf_profile_id", "type": "path", "expr": "$.tags.sf_profile_id" }, + { "name": "sent_pkts", "type": "path", "expr": "$.fields.sent_pkts" }, + { "name": "sent_bytes", "type": "path", "expr": "$.fields.sent_bytes" }, + { "name": "recv_pkts", "type": "path", "expr": "$.fields.recv_pkts" }, + { "name": "recv_bytes", "type": "path", "expr": "$.fields.recv_bytes" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sent_pkts", "fieldName" : "sent_pkts" }, + { "type" : "longSum", "name" : "sent_bytes", "fieldName" : "sent_bytes" }, + { "type" : "longSum", "name" : "recv_pkts", "fieldName" : "recv_pkts" }, + { "type" : "longSum", "name" : "recv_bytes", "fieldName" : "recv_bytes" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "sent_pkts", "expression": "nvl(sent_pkts, 0)"}, + {"type": "expression", "name": "sent_bytes", "expression": "nvl(sent_bytes, 0)"}, + {"type": "expression", "name": "recv_pkts", "expression": "nvl(recv_pkts, 0)"}, + {"type": "expression", "name": "recv_bytes", "expression": "nvl(recv_bytes, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "service_chaining_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json new file mode 100644 index 0000000..61e5b12 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json @@ -0,0 +1,74 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "service_function_status", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { "name": "vsys_id", "type": "long" }, + { "name": "sf_profile_id", "type": "long" } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "sf_profile_id", "type": "path", "expr": "$.tags.sf_profile_id" }, + { "name": "sf_status", "type": "path", "expr": "$.fields.sf_status" }, + { "name": "sf_latency_us", "type": "path", "expr": "$.fields.sf_latency_us" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longMax", "name" : "sf_latency_us", "fieldName" : "sf_latency_us" }, + { "type" : "longMax", "name" : "sf_status", "fieldName" : "sf_status" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "sf_latency_us", "expression": "nvl(sf_latency_us, 0)"}, + {"type": "expression", "name": "sf_status", "expression": "nvl(sf_status, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "service_function_status" } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json new file mode 100644 index 0000000..8c15a37 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json @@ -0,0 +1,367 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "statistics_rule", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { + "name": "vsys_id", + "type": "long" + }, + { + "name": "rule_id", + "type": "long" + }, + { + "name": "template_id", + "type": "long" + }, + { + "name": "chart_id", + "type": "long" + }, + { + "name": "version", + "type": "long" + }, + "client_ip_object", + "server_ip_object", + "fqdn_category", + "client_ip", + "server_ip", + "server_fqdn", + "server_domain", + "application" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { + "name": "vsys_id", + "type": "path", + "expr": "$.tags.vsys_id" + }, + { + "name": "data_center", + "type": "path", + "expr": "$.tags.data_center" + }, + { + "name": "device_group", + "type": "path", + "expr": "$.tags.device_group" + }, + { + "name": "device_id", + "type": "path", + "expr": "$.tags.device_id" + }, + { + "name": "rule_id", + "type": "path", + "expr": "$.tags.rule_id" + }, + { + "name": "template_id", + "type": "path", + "expr": "$.tags.template_id" + }, + { + "name": "chart_id", + "type": "path", + "expr": "$.tags.chart_id" + }, + { + "name": "version", + "type": "path", + "expr": "$.tags.version" + }, + { + "name": "client_ip_object", + "type": "path", + "expr": "$.tags.client_ip_object" + }, + { + "name": "server_ip_object", + "type": "path", + "expr": "$.tags.server_ip_object" + }, + { + "name": "fqdn_category", + "type": "path", + "expr": "$.tags.fqdn_category" + }, + { + "name": "client_ip", + "type": "path", + "expr": "$.tags.client_ip" + }, + { + "name": "server_ip", + "type": "path", + "expr": "$.tags.server_ip" + }, + { + "name": "server_fqdn", + "type": "path", + "expr": "$.tags.server_fqdn" + }, + { + "name": "server_domain", + "type": "path", + "expr": "$.tags.server_domain" + }, + { + "name": "application", + "type": "path", + "expr": "$.tags.application" + }, + { + "name": "in_bytes", + "type": "path", + "expr": "$.fields.in_bytes" + }, + { + "name": "out_bytes", + "type": "path", + "expr": "$.fields.out_bytes" + }, + { + "name": "bytes", + "type": "path", + "expr": "$.fields.bytes" + }, + { + "name": "new_c2s_flows", + "type": "path", + "expr": "$.fields.new_c2s_flows" + }, + { + "name": "new_s2c_flows", + "type": "path", + "expr": "$.fields.new_s2c_flows" + }, + { + "name": "new_in_sessions", + "type": "path", + "expr": "$.fields.new_in_sessions" + }, + { + "name": "new_out_sessions", + "type": "path", + "expr": "$.fields.new_out_sessions" + }, + { + "name": "sessions", + "type": "path", + "expr": "$.fields.sessions" + }, + { + "name": "new_unestablished_sessions", + "type": "path", + "expr": "$.fields.new_unestablished_sessions" + }, + { + "name": "syn_pkts", + "type": "path", + "expr": "$.fields.syn_pkts" + }, + { + "name": "session_identifier_sketch", + "type": "path", + "expr": "$.fields.session_identifier_sketch" + }, + { + "name": "client_ip_sketch", + "type": "path", + "expr": "$.fields.client_ip_sketch" + }, + { + "name": "server_ip_sketch", + "type": "path", + "expr": "$.fields.server_ip_sketch" + }, + { + "name": "in_latency_ms_sketch", + "type": "path", + "expr": "$.fields.in_latency_ms_sketch" + }, + { + "name": "out_latency_ms_sketch", + "type": "path", + "expr": "$.fields.out_latency_ms_sketch" + }, + { + "name": "latency_ms_sketch", + "type": "path", + "expr": "$.fields.latency_ms_sketch" + } + ] + } + } + }, + "metricsSpec": [ + { + "type": "HLLDSketchMerge", + "name": "session_identifier_sketch", + "fieldName": "session_identifier_sketch" + }, + { + "type": "HLLDSketchMerge", + "name": "client_ip_sketch", + "fieldName": "client_ip_sketch" + },{ + "type": "HLLDSketchMerge", + "name": "server_ip_sketch", + "fieldName": "server_ip_sketch" + }, + { + "type": "HdrHistogramSketchMerge", + "name": "in_latency_ms_sketch", + "fieldName": "in_latency_ms_sketch", + "lowestDiscernibleValue": "1", + "highestTrackableValue": "2", + "numberOfSignificantValueDigits": "3", + "autoResize": "true" + }, + { + "type": "HdrHistogramSketchMerge", + "name": "out_latency_ms_sketch", + "fieldName": "out_latency_ms_sketch", + "lowestDiscernibleValue": "1", + "highestTrackableValue": "2", + "numberOfSignificantValueDigits": "3", + "autoResize": "true" + }, + { + "type": "HdrHistogramSketchMerge", + "name": "latency_ms_sketch", + "fieldName": "latency_ms_sketch", + "lowestDiscernibleValue": "1", + "highestTrackableValue": "2", + "numberOfSignificantValueDigits": "3", + "autoResize": "true" + }, + { + "type": "longSum", + "name": "in_bytes", + "fieldName": "in_bytes" + }, + { + "type": "longSum", + "name": "out_bytes", + "fieldName": "out_bytes" + }, + { + "type": "longSum", + "name": "bytes", + "fieldName": "bytes" + }, + { + "type": "longSum", + "name": "new_c2s_flows", + "fieldName": "new_c2s_flows" + }, + { + "type": "longSum", + "name": "new_s2c_flows", + "fieldName": "new_s2c_flows" + }, + { + "type": "longSum", + "name": "new_in_sessions", + "fieldName": "new_in_sessions" + }, + { + "type": "longSum", + "name": "new_out_sessions", + "fieldName": "new_out_sessions" + }, + { + "type": "longSum", + "name": "sessions", + "fieldName": "sessions" + }, + { + "type": "longSum", + "name": "new_unestablished_sessions", + "fieldName": "new_unestablished_sessions" + }, + { + "type": "longSum", + "name": "syn_pkts", + "fieldName": "syn_pkts" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "period", + "period": "PT15S" + }, + "rollup": true + }, + "transformSpec": { + "transforms": [ + {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, + {"type": "expression", "name": "client_ip_object", "expression": "STRING_TO_ARRAY(client_ip_object,',')"}, + {"type": "expression", "name": "server_ip_object", "expression": "STRING_TO_ARRAY(server_ip_object,',')"}, + {"type": "expression", "name": "fqdn_category", "expression": "STRING_TO_ARRAY(fqdn_category,',')"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"}, + {"type": "expression", "name": "new_c2s_flows", "expression": "nvl(new_c2s_flows, 0)"}, + {"type": "expression", "name": "new_s2c_flows", "expression": "nvl(new_s2c_flows, 0)"}, + {"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"}, + {"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "new_unestablished_sessions", "expression": "nvl(new_unestablished_sessions, 0)"}, + {"type": "expression", "name": "syn_pkts", "expression": "nvl(syn_pkts, 0)"} + ], + "filter": { + "type": "selector", + "dimension": "name", + "value": "statistics_rule" + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "maxRowsInMemory": 400000, + "maxBytesInMemory": -1, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "STATISTICS-RULE-METRIC", + "taskCount": 2, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + }, + "context":{ + "druid.indexer.runner.javaOpts": "{{ druid.index_kafka_statistics_rule['druid.indexer.runner.javaOpts'] }}" + } +} + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json new file mode 100644 index 0000000..f431e0f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json @@ -0,0 +1,159 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "statistics_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { + "name": "vsys_id", + "type": "long" + }, + { + "name": "rule_id", + "type": "long" + } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { + "name": "vsys_id", + "type": "path", + "expr": "$.tags.vsys_id" + }, + { + "name": "data_center", + "type": "path", + "expr": "$.tags.data_center" + }, + { + "name": "device_group", + "type": "path", + "expr": "$.tags.device_group" + }, + { + "name": "device_id", + "type": "path", + "expr": "$.tags.device_id" + }, + { + "name": "rule_id", + "type": "path", + "expr": "$.tags.rule_id" + }, + { + "name": "in_bytes", + "type": "path", + "expr": "$.fields.in_bytes" + }, + { + "name": "out_bytes", + "type": "path", + "expr": "$.fields.out_bytes" + }, + { + "name": "hit_count", + "type": "path", + "expr": "$.fields.hit_count" + }, + { + "name": "in_pkts", + "type": "path", + "expr": "$.fields.in_pkts" + }, + { + "name": "out_pkts", + "type": "path", + "expr": "$.fields.out_pkts" + } + ] + } + } + }, + "metricsSpec": [ + { + "type": "longSum", + "name": "in_bytes", + "fieldName": "in_bytes" + }, + { + "type": "longSum", + "name": "out_bytes", + "fieldName": "out_bytes" + }, + { + "type": "longSum", + "name": "hit_count", + "fieldName": "hit_count" + }, + { + "type": "longSum", + "name": "in_pkts", + "fieldName": "in_pkts" + }, + { + "type": "longSum", + "name": "out_pkts", + "fieldName": "out_pkts" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "period", + "period": "PT1S" + }, + "rollup": true + }, + "transformSpec": { + "transforms": [ + {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { + "type": "selector", + "dimension": "name", + "value": "statistics_rule_hits" + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json new file mode 100644 index 0000000..2b405c2 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json @@ -0,0 +1,52 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sys_storage_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "log_type", + "data_center", + { "name": "used_size", "type": "long" }, + { "name": "max_size", "type": "long" }, + { "name": "aggregate_size", "type": "long" }, + { "name": "last_storage", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SYS-STORAGE", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json new file mode 100644 index 0000000..4a8749f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json @@ -0,0 +1,104 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_client_ips", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "client_ip", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "client_ip", "type": "path", "expr": "$.tags.client_ip" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"packets_top_client_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_client_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_client_ip" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json new file mode 100644 index 0000000..5ff580e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_external_ips", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "external_ip", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "external_ip", "type": "path", "expr": "$.tags.external_ip" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_external_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_external_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_external_ip" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json new file mode 100644 index 0000000..8767165 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_internal_ips", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "internal_ip", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "internal_ip", "type": "path", "expr": "$.tags.internal_ip" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_internal_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_internal_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_internal_ip" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json new file mode 100644 index 0000000..bd5b144 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_server_domains", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "domain", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "domain", "type": "path", "expr": "$.tags.domain" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"packets_top_server_domain" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_server_domain" + }, + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_server_domain" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json new file mode 100644 index 0000000..4bc42af --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_server_fqdns", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "fqdn", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "fqdn", "type": "path", "expr": "$.tags.fqdn" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_server_fqdn" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_server_fqdn" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_server_fqdn" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json new file mode 100644 index 0000000..1f05adf --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_server_ips", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "server_ip", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "server_ip", "type": "path", "expr": "$.tags.server_ip" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_server_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_server_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_server_ip" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json new file mode 100644 index 0000000..8bc5e37 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_subscribers", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "subscriber_id", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "subscriber_id", "type": "path", "expr": "$.tags.subscriber_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_subscriber_id" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_subscriber_id" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_subscriber_id" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json new file mode 100644 index 0000000..6013977 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_general_stat", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { "name": "vsys_id", "type": "long" } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "active_sessions", "type": "path", "expr": "$.fields.active_sessions" }, + { "name": "closed_sessions", "type": "path", "expr": "$.fields.closed_sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "asymmetric_c2s_flows", "type": "path", "expr": "$.fields.asymmetric_c2s_flows" }, + { "name": "asymmetric_s2c_flows", "type": "path", "expr": "$.fields.asymmetric_s2c_flows" }, + { "name": "tcp_syn_pkts", "type": "path", "expr": "$.fields.tcp_syn_pkts" }, + { "name": "tcp_ack_pkts", "type": "path", "expr": "$.fields.tcp_ack_pkts" }, + { "name": "tcp_fin_pkts", "type": "path", "expr": "$.fields.tcp_fin_pkts" }, + { "name": "tcp_rst_pkts", "type": "path", "expr": "$.fields.tcp_rst_pkts" } + ] + } + } + }, + "metricsSpec":[ + { "type":"longSum", "name":"sessions", "fieldName":"sessions" }, + { "type":"longMax", "name":"active_sessions", "fieldName":"active_sessions" }, + { "type":"longSum", "name":"closed_sessions", "fieldName":"closed_sessions" }, + { "type":"longSum", "name":"in_bytes", "fieldName":"in_bytes" }, + { "type":"longSum", "name":"out_bytes", "fieldName":"out_bytes" }, + { "type":"longSum", "name":"in_pkts", "fieldName":"in_pkts" }, + { "type":"longSum", "name":"out_pkts", "fieldName":"out_pkts" }, + { "type":"longSum", "name":"asymmetric_c2s_flows", "fieldName":"asymmetric_c2s_flows" }, + { "type":"longSum", "name":"asymmetric_s2c_flows", "fieldName":"asymmetric_s2c_flows" }, + { "type":"longSum", "name":"tcp_syn_pkts", "fieldName":"tcp_syn_pkts" }, + { "type":"longSum", "name":"tcp_ack_pkts", "fieldName":"tcp_ack_pkts" }, + { "type":"longSum", "name":"tcp_fin_pkts", "fieldName":"tcp_fin_pkts" }, + { "type":"longSum", "name":"tcp_rst_pkts", "fieldName":"tcp_rst_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "active_sessions", "expression": "nvl(active_sessions, 0)"}, + {"type": "expression", "name": "closed_sessions", "expression": "nvl(closed_sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}, + {"type": "expression", "name": "asymmetric_c2s_flows", "expression": "nvl(asymmetric_c2s_flows, 0)"}, + {"type": "expression", "name": "asymmetric_s2c_flows", "expression": "nvl(asymmetric_s2c_flows, 0)"}, + {"type": "expression", "name": "tcp_syn_pkts", "expression": "nvl(tcp_syn_pkts, 0)"}, + {"type": "expression", "name": "tcp_ack_pkts", "expression": "nvl(tcp_ack_pkts, 0)"}, + {"type": "expression", "name": "tcp_fin_pkts", "expression": "nvl(tcp_fin_pkts, 0)"}, + {"type": "expression", "name": "tcp_rst_pkts", "expression": "nvl(tcp_rst_pkts, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "traffic_general_stat" } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "NETWORK-TRAFFIC-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json new file mode 100644 index 0000000..a5a644d --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json @@ -0,0 +1,101 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_shaping_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + {"name": "rule_id","type": "long"}, + {"name": "profile_id","type": "long"}, + {"name": "priority","type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "in_drop_pkts", "type": "path", "expr": "$.fields.in_drop_pkts" }, + { "name": "out_drop_pkts", "type": "path", "expr": "$.fields.out_drop_pkts" }, + { "name": "in_max_latency_us", "type": "path", "expr": "$.fields.in_max_latency_us" }, + { "name": "out_max_latency_us", "type": "path", "expr": "$.fields.out_max_latency_us" }, + { "name": "in_queue_len", "type": "path", "expr": "$.fields.in_queue_len" }, + { "name": "out_queue_len", "type": "path", "expr": "$.fields.out_queue_len" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "profile_id", "type": "path", "expr": "$.tags.profile_id" }, + { "name": "priority", "type": "path", "expr": "$.tags.priority" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }, + { "type" : "longSum", "name" : "in_drop_pkts", "fieldName" : "in_drop_pkts" }, + { "type" : "longSum", "name" : "out_drop_pkts", "fieldName" : "out_drop_pkts" }, + { "type" : "longMax", "name" : "in_max_latency_us", "fieldName" : "in_max_latency_us" }, + { "type" : "longMax", "name" : "out_max_latency_us", "fieldName" : "out_max_latency_us" }, + { "type" : "longMax", "name" : "in_queue_len", "fieldName" : "in_queue_len" }, + { "type" : "longMax", "name" : "out_queue_len", "fieldName" : "out_queue_len" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}, + {"type": "expression", "name": "in_drop_pkts", "expression": "nvl(in_drop_pkts, 0)"}, + {"type": "expression", "name": "out_drop_pkts", "expression": "nvl(out_drop_pkts, 0)"}, + {"type": "expression", "name": "in_max_latency_us", "expression": "nvl(in_max_latency_us, 0)"}, + {"type": "expression", "name": "out_max_latency_us", "expression": "nvl(out_max_latency_us, 0)"}, + {"type": "expression", "name": "in_queue_len", "expression": "nvl(in_queue_len, 0)"}, + {"type": "expression", "name": "out_queue_len", "expression": "nvl(out_queue_len, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "traffic_shaping_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json new file mode 100644 index 0000000..5541452 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json @@ -0,0 +1,61 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_top_destination_ip_metrics_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "sketch_start_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "destination_ip", + "attack_type", + { "name": "vsys_id", "type": "long" }, + { "type": "long", "name": "partition_num" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"}, + { "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" }, + { "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT10M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression","name": "vsys_id","expression": "nvl(vsys_id,1)"}, + {"type": "expression", "name": "session_rate", "expression": "nvl(session_rate, 0)"}, + {"type": "expression", "name": "packet_rate", "expression": "nvl(packet_rate, 0)"}, + {"type": "expression", "name": "bit_rate", "expression": "nvl(bit_rate, 0)"} + ] + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-DESTINATION-IP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/vars/main.yml b/tsg-olap/parcels/roles/initialization/druid/vars/main.yml new file mode 100644 index 0000000..3fc948a --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/vars/main.yml @@ -0,0 +1,11 @@ +container_name: druid + +backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/{{ container_name }}" + +backup_items: + - conf + - docker-compose.yml + - init + - monitor + - topology + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml new file mode 100644 index 0000000..2c5336b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml @@ -0,0 +1,28 @@ +- name: Create buckets + uri: + url: "http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/{{ item }}" + method: PUT + headers: + token: "{{ hos_token }}" + return_content: yes + retries: 3 + delay: 3 + loop: "{{ init_buckets }}" + run_once: true + +- name: Check if buckets exist + uri: + url: "http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/" + method: GET + headers: + token: "{{ hos_token }}" + return_content: yes + register: bucket_info + retries: 3 + delay: 3 + +- name: Check create bucket result + fail: + msg: "{{ item }} was not created successfully. Please retry." + when: "'{{ item }}' not in bucket_info.content" + loop: "{{ init_buckets }}" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml new file mode 100644 index 0000000..125d374 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml @@ -0,0 +1,5 @@ +- include: init.yml + when: operation == 'install' + +- include: upgrade.yml + when: operation == 'upgrade' diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml new file mode 100644 index 0000000..cd21505 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml @@ -0,0 +1,2 @@ +--- + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml new file mode 100644 index 0000000..9203abd --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml @@ -0,0 +1,6 @@ +init_buckets: + - troubleshooting_file_bucket + - assessment_file_bucket + - knowledge_base_bucket + - traffic_file_bucket + - report_snapshot_bucket diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/handlers/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/handlers/main.yml new file mode 100644 index 0000000..6df35ce --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/handlers/main.yml @@ -0,0 +1,41 @@ +- name: Stop Container + docker_container: + name: '{{ image_name }}' + state: absent + force_kill: true + +- name: Remove Image + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag }}' + state: absent + force_absent: true + +- name: Copy Image + copy: + src: 'files/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}' + force: yes + +- name: Unarchive Image + unarchive: + src: 'files/{{ image_name }}-{{ image_tag }}.tar.gz' + dest: '{{ deploy_dir }}/{{ container_name }}' + force: yes + +- name: Load Image + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag }}' + load_path: '{{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar' + source: load + force_tag: yes + force_source: yes + timeout: 300 + +- name: Start Container + docker_compose: + project_src: '{{ deploy_dir }}/{{ container_name }}' + +- name: Start Nginx + shell: 'docker restart galaxy-gateway-nginx' diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml new file mode 100644 index 0000000..3a66bc3 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml @@ -0,0 +1,51 @@ +- name: Check if admin backup directory exists + stat: + path: "{{ backup_path }}/{{ admin_container_name }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}/{{ admin_container_name }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/{{ admin_container_name }}/{{ item }}" + dest: "{{ backup_path }}/{{ admin_container_name }}/" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}/{{ admin_container_name }}" + state: absent + when: exist_status.stat.exists == false + +- name: Check if executor backup directory exists + stat: + path: "{{ backup_path }}/{{ executor_container_name }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}/{{ executor_container_name }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/{{ executor_container_name }}/{{ item }}" + dest: "{{ backup_path }}/{{ executor_container_name }}/" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}/{{ executor_container_name }}" + state: absent + when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml new file mode 100644 index 0000000..14e1d40 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml @@ -0,0 +1,12 @@ +- name: Check the health status of galaxy-job-admin + uri: + url: http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin/login + method: POST + body_format: form-urlencoded + body: + userName: admin + pin: galaxy_2019 + +- name: Check that the galaxy-job-admin is accessible + uri: + url: "http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin/testCase/logPolicyTest" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml new file mode 100644 index 0000000..14cfe28 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml @@ -0,0 +1,52 @@ +- name: Set container_name + set_fact: + container_name: "{{ admin_container_name }}" + image_name: "{{ admin_image_name }}" + image_tag: "{{ admin_image_tag }}" + +- name: Creating directory + file: + state: directory + path: '{{ deploy_dir }}/{{ container_name }}/config' + +- name: Copying image to {{ deploy_dir }}/{{ container_name }}/ + copy: + src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}/' + force: true + notify: + - Load Image + - Start Container + +- name: Copying {{ container_name }} config + template: + src: "admin/{{ item }}.j2" + dest: '{{ deploy_dir }}/{{ container_name }}/config/{{ item }}' + force: yes + loop: + - application.properties + - logback.xml + notify: + - Start Container + +- name: Copying {{ container_name }} docker-compose.yml + template: + src: "admin/docker-compose.yml.j2" + dest: '{{ deploy_dir }}/{{ container_name }}/docker-compose.yml' + force: yes + notify: + - Start Container + +- name: Copying {{ container_name }} nginx config + template: + src: "admin/{{ container_name }}.conf.j2" + dest: '{{ deploy_dir }}/galaxy-gateway-nginx/conf/includes/{{ container_name }}.conf' + force: yes + notify: + - Start Nginx + delegate_to: "{{ item }}" + run_once: true + loop: "{{ groups.loadbalancer[:2] }}" + +- meta: flush_handlers + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml new file mode 100644 index 0000000..7ea6d93 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml @@ -0,0 +1,45 @@ +- name: Setting variable name + set_fact: + container_name: "{{ executor_container_name }}" + image_name: "{{ executor_image_name }}" + image_tag: "{{ executor_image_tag }}" + +- name: Creating {{ container_name }} directory + file: + state: directory + path: '{{ deploy_dir }}/{{ container_name }}/{{ item }}' + loop: + - "init" + - "config" + +- name: Copying image to {{ deploy_dir }}/{{ container_name }}/ + copy: + src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}/' + force: true + notify: + - Load Image + - Start Container + +- name: Copying {{ container_name }} config + template: + src: "executor/{{ item }}.j2" + dest: '{{ deploy_dir }}/{{ container_name }}/config/{{ item }}' + force: yes + loop: + - application.properties + - logback.xml + notify: + - Start Container + +- name: Copying {{ container_name }} docker-compose.yml + template: + src: "executor/docker-compose.yml.j2" + dest: '{{ deploy_dir }}/{{ container_name }}/docker-compose.yml' + force: yes + notify: + - Start Container + + +- meta: flush_handlers + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml new file mode 100644 index 0000000..d3385f7 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml @@ -0,0 +1,53 @@ +- name: Creating directory + file: + state: directory + path: '{{ deploy_dir }}/{{ item }}/init' + loop: + - 'galaxy-job-admin' + - 'galaxy-job-executor' + +- name: Copying admin init files + template: + src: "{{ item.src }}" + dest: "{{ deploy_dir }}/{{ admin_container_name }}/init/{{ item.dest }}" + mode: "{{ item.mode }}" + force: true + loop: + - { src: "push_config.sh.j2", dest: "push_config.sh", mode: "0755" } + - { src: "admin/xxl_job.sql.j2", dest: "xxl_job.sql", mode: "0644" } + - { src: 'admin/create_mariadb_tables.sh.j2', dest: 'create_mariadb_tables.sh', mode: "0755" } + - { src: "admin/galaxy-job-admin.properties.j2", dest: "galaxy-job-admin.properties", mode: "0644" } + +- name: Copying mysql Client to /usr/bin/ + copy: + src: 'files/mysql' + dest: '/usr/bin/' + mode: 0755 + force: true + +- name: Creating table and init jobs + shell: cd {{ deploy_dir }}/{{ admin_container_name }}/init && sh create_mariadb_tables.sh + run_once: true + when: operation == 'install' + +- name: Copying executor init files + template: + src: "{{ item.src }}" + dest: "{{ deploy_dir }}/{{ executor_container_name }}/init/{{ item.dest }}" + mode: "{{ item.mode }}" + force: true + loop: + - { src: "push_config.sh.j2", dest: "push_config.sh", mode: "0755" } + - { src: "executor/galaxy-job-executor.properties.j2", dest: "galaxy-job-executor.properties", mode: "0644" } + +- name: Pushing Galaxy-job-service admin config + shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && sh push_config.sh {{ item.conf }}' + register: post_result + until: post_result.stdout == '1' + retries: 3 + delay: 3 + failed_when: post_result.stdout != '1' + loop: + - { src: "{{ admin_container_name }}" , conf: "galaxy-job-admin.properties" } + - { src: "{{ executor_container_name }}" , conf: "galaxy-job-executor.properties" } + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml new file mode 100644 index 0000000..a1d5b98 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml @@ -0,0 +1,22 @@ +- block: + - include: uninstall-admin.yml + - include: uninstall-executor.yml + - include: init.yml + - include: deploy-admin.yml + - include: deploy-executor.yml + when: operation == 'install' + +- block: + - include: check.yml + - include: backup.yml + - include: init.yml + - include: upgrade.yml + - include: deploy-executor.yml + - include: check.yml + when: operation == 'upgrade' + +- block: + - include: uninstall-admin.yml + - include: uninstall-executor.yml + when: operation == 'uninstall' + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml new file mode 100644 index 0000000..2ebe9c8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml @@ -0,0 +1,18 @@ +- name: Stopping and removing {{ admin_container_name }} container + docker_container: + name: '{{ admin_container_name }}' + state: absent + force_kill: true + +- name: Removing old {{ admin_image_name }} image + docker_image: + name: '{{ admin_image_name }}' + tag: '{{ admin_image_tag }}' + state: absent + force_absent: true + +- name: Ansible delete old {{ deploy_dir }}/{{ admin_container_name }} + file: + path: '{{ deploy_dir }}/{{ admin_container_name }}' + state: absent + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml new file mode 100644 index 0000000..3e4deb8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml @@ -0,0 +1,18 @@ +- name: Stopping and removing {{ executor_container_name }} container + docker_container: + name: '{{ executor_container_name }}' + state: absent + force_kill: true + +- name: Removing old {{ executor_image_name }} image + docker_image: + name: '{{ executor_image_name }}' + tag: '{{ executor_image_tag }}' + state: absent + force_absent: true + +- name: Ansible delete old {{ deploy_dir }}/{{ executor_container_name }} + file: + path: '{{ deploy_dir }}/{{ executor_container_name }}' + state: absent + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml new file mode 100644 index 0000000..5da8917 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml @@ -0,0 +1,54 @@ +- name: Creating config tmp directory + file: + state: directory + path: "/tmp/{{ item }}" + loop: + - "{{ admin_container_name }}" + - "{{ executor_container_name }}" + +- name: Creating upgrade directory + file: + state: directory + path: "{{ deploy_dir }}/{{ executor_container_name }}/upgrade/upgrade-sql" + +- name: Fetching config files + fetch: + src: "{{ deploy_dir }}/{{ item.name }}/{{ item.config }}" + dest: "/tmp/{{ item.name }}/{{ item.config }}" + flat: yes + loop: + - { name: "{{ admin_container_name }}", config: 'docker-compose.yml' } + - { name: "{{ executor_container_name }}", config: 'docker-compose.yml' } + +- name: Loading variable by admin docker-compose.yml + include_vars: + file: "/tmp/{{ admin_container_name }}/docker-compose.yml" + name: admin_docker_config + +- name: Loading variable by executor docker-compose.yml + include_vars: + file: "/tmp/{{ executor_container_name }}/docker-compose.yml" + name: executor_docker_config + +- name: Set variable + set_fact: + galaxy_job_service: + admin_java_opts: "{{ admin_docker_config['services']['galaxy-job-admin']['environment']['JAVA_OPTS'] }}" + executor_java_opts: "{{ executor_docker_config['services']['galaxy-job-executor']['environment']['JAVA_OPTS'] }}" + +- name: Copying upgrade sql file + copy: + src: 'files/upgrade-sql' + dest: '{{ deploy_dir }}/{{ executor_container_name }}/upgrade/' + force: true + +- name: Copying upgrade-mariadb-data.sh template + template: + src: "upgrade-mariadb-data.sh.j2" + dest: "{{ deploy_dir }}/{{ executor_container_name }}/upgrade/upgrade-mariadb-data.sh" + mode: 0755 + force: true + +- name: Upgrade mariadb data + shell: cd {{ deploy_dir }}/{{ executor_container_name }}/upgrade && sh upgrade-mariadb-data.sh + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 new file mode 100644 index 0000000..6ea1881 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 @@ -0,0 +1,23 @@ +#配置集的配置格式 +nacos.config.type=properties +#配置中心地址 +nacos.config.server-addr={{ vrrp_instance.default.virtual_ipaddress }}:8848 +#命名空间 +nacos.config.namespace={{ config_namespace }} +#数据集ID +nacos.config.data-id=galaxy-job-admin.properties +#开启自动刷新 +nacos.config.auto-refresh=true +#配置对应的分组 +nacos.config.group=Galaxy +#Nacos认证用户 +nacos.config.username=nacos +#Nacos认证密码 +nacos.config.password=nacos +#开启配置预加载功能 +nacos.config.bootstrap.enable=true +#开启Nacos支持日志级别的加载时机 +nacos.config.bootstrap.log-enable=true + +## log config +logging.config=./config/logback.xml diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 new file mode 100644 index 0000000..392997f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash + +mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} -e "create database if not exists {{ mariadb_job_database }} default character set utf8mb4 collate utf8mb4_general_ci;" + +mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} {{ mariadb_job_database }} < {{ deploy_dir }}/{{ admin_container_name }}/init/xxl_job.sql + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 new file mode 100644 index 0000000..be4abb8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 @@ -0,0 +1,15 @@ +version: '3' +services: + galaxy-job-admin: + image: {{ image_name }}:{{ image_tag }} + container_name: galaxy-job-admin + environment: + JAVA_OPTS: "{{ galaxy_job_service.admin_java_opts }}" + ports: + - "8181:8181" + volumes: + - "{{ deploy_dir }}/{{ container_name }}/logs:/logs" + - "{{ deploy_dir }}/{{ container_name }}/data:/data" + - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-admin/config" + restart: always + network_mode: "host" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 new file mode 100644 index 0000000..bee9eea --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 @@ -0,0 +1,16 @@ +upstream jobAdmin { + +{% for host in groups['galaxy_job_service'][:2] %} + server {{ host }}:8184; +{% endfor %} + +} + +server { + listen 8181; + server_name localhost; + location / { + proxy_pass http://jobAdmin; + } +} + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 new file mode 100644 index 0000000..082e719 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 @@ -0,0 +1,73 @@ +### web +server.port=8184 +server.servlet.context-path=/xxl-job-admin +spring.application.name=galaxy-job-admin +### actuator +management.server.servlet.context-path=/actuator +management.health.mail.enabled=false +management.endpoints.web.exposure.include=* +#详细的应用健康信息 prometheus +management.endpoint.health.show-details=always + +management.endpoint.metrics.enabled=true +management.endpoint.prometheus.enabled=true +management.metrics.export.prometheus.enabled=true +management.metrics.tags.application=${spring.application.name} +management.metrics.tags.module=${spring.application.name} + + +### resources +spring.mvc.servlet.load-on-startup=0 +spring.mvc.static-path-pattern=/static/** +spring.resources.static-locations=classpath:/static/ + +### freemarker +spring.freemarker.templateLoaderPath=classpath:/templates/ +spring.freemarker.suffix=.ftl +spring.freemarker.charset=UTF-8 +spring.freemarker.request-context-attribute=request +spring.freemarker.settings.number_format=0.########## + +### mybatis +mybatis.mapper-locations=classpath:/mybatis-mapper/*Mapper.xml +#mybatis.type-aliases-package=com.xxl.job.admin.core.model + +### xxl-job, datasource +spring.datasource.url=jdbc:mysql://{{ vrrp_instance.default.virtual_ipaddress }}:3306/{{ mariadb_job_database }}?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&serverTimezone=UTC +spring.datasource.username=ENC(63aTpwv2vH0vPikW+3Jjig==) +spring.datasource.password=ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y) +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver + +### datasource-pool +spring.datasource.type=com.zaxxer.hikari.HikariDataSource +spring.datasource.hikari.minimum-idle=10 +spring.datasource.hikari.maximum-pool-size=100 +spring.datasource.hikari.auto-commit=true +spring.datasource.hikari.idle-timeout=30000 +spring.datasource.hikari.pool-name=HikariCP +spring.datasource.hikari.max-lifetime=900000 +spring.datasource.hikari.connection-timeout=30000 +spring.datasource.hikari.connection-test-query=SELECT 1 + +### xxl-job, email +spring.mail.host=smtp.qq.com +spring.mail.port=25 +spring.mail.username=xxx@qq.com +spring.mail.password=xxx +spring.mail.properties.mail.smtp.auth=true +spring.mail.properties.mail.smtp.starttls.enable=true +spring.mail.properties.mail.smtp.starttls.required=true +spring.mail.properties.mail.smtp.socketFactory.class=javax.net.ssl.SSLSocketFactory + +### xxl-job, access token +xxl.job.accessToken= + +### xxl-job, i18n (default is zh_CN, and you can choose "zh_CN", "zh_TC" and "en") +xxl.job.i18n=zh_CN + +## xxl-job, triggerpool max size +xxl.job.triggerpool.fast.max=200 +xxl.job.triggerpool.slow.max=100 + +### xxl-job, log retention days +xxl.job.logretentiondays=30 diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 new file mode 100644 index 0000000..dd729b0 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + ${LOG_PATTERN} + + + + + ${LOG_PATH}/${LOG_FILE_NAME}.log + + ALL + + + ${LOG_PATTERN} + + + + ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz + + ${LOG_DAYS} + ${LOG_SIZE} + + + + + + + + + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 new file mode 100644 index 0000000..f3562c8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 @@ -0,0 +1,195 @@ +/* +Navicat MySQL Data Transfer + +Target Server Type : MYSQL +Target Server Version : 50505 +File Encoding : 65001 + +Date: 2021-09-23 14:05:11 +*/ + +SET FOREIGN_KEY_CHECKS=0; + +-- ---------------------------- +-- Table structure for `xxl_job_group` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_group`; +CREATE TABLE `xxl_job_group` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `app_name` varchar(64) NOT NULL COMMENT '执行器AppName', + `title` varchar(64) NOT NULL COMMENT '执行器名称', + `address_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '执行器地址类型：0=自动注册、1=手动录入', + `address_list` varchar(512) DEFAULT NULL COMMENT '执行器地址列表，多地址逗号分隔', + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Table structure for `xxl_job_lock` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_lock`; +CREATE TABLE `xxl_job_lock` ( + `lock_name` varchar(50) NOT NULL COMMENT '锁名称', + PRIMARY KEY (`lock_name`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_lock +-- ---------------------------- +INSERT INTO `xxl_job_lock` VALUES ('schedule_lock'); + +-- ---------------------------- +-- Table structure for `xxl_job_log` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_log`; +CREATE TABLE `xxl_job_log` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_group` int(11) NOT NULL COMMENT '执行器主键ID', + `job_id` int(11) NOT NULL COMMENT '任务，主键ID', + `executor_address` varchar(255) DEFAULT NULL COMMENT '执行器地址，本次执行的地址', + `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', + `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', + `executor_sharding_param` varchar(20) DEFAULT NULL COMMENT '执行器任务分片参数，格式如 1/2', + `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', + `trigger_time` datetime DEFAULT NULL COMMENT '调度-时间', + `trigger_code` int(11) NOT NULL COMMENT '调度-结果', + `trigger_msg` text DEFAULT NULL COMMENT '调度-日志', + `handle_time` datetime DEFAULT NULL COMMENT '执行-时间', + `handle_code` int(11) NOT NULL COMMENT '执行-状态', + `handle_msg` text DEFAULT NULL COMMENT '执行-日志', + `alarm_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '告警状态：0-默认、1-无需告警、2-告警成功、3-告警失败', + PRIMARY KEY (`id`) USING BTREE, + KEY `I_trigger_time` (`trigger_time`) USING BTREE, + KEY `I_handle_code` (`handle_code`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=2077229 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_log +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_logglue` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_logglue`; +CREATE TABLE `xxl_job_logglue` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `job_id` int(11) NOT NULL COMMENT '任务，主键ID', + `glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型', + `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', + `glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注', + `add_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_logglue +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_log_report` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_log_report`; +CREATE TABLE `xxl_job_log_report` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `trigger_day` datetime DEFAULT NULL COMMENT '调度-时间', + `running_count` int(11) NOT NULL DEFAULT 0 COMMENT '运行中-日志数量', + `suc_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行成功-日志数量', + `fail_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行失败-日志数量', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=786 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_log_report +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_registry` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_registry`; +CREATE TABLE `xxl_job_registry` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `registry_group` varchar(50) NOT NULL, + `registry_key` varchar(255) NOT NULL, + `registry_value` varchar(255) NOT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=276 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Table structure for `xxl_job_user` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_user`; +CREATE TABLE `xxl_job_user` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `username` varchar(50) NOT NULL COMMENT '账号', + `password` varchar(50) NOT NULL COMMENT '密码', + `role` tinyint(4) NOT NULL COMMENT '角色：0-普通用户、1-管理员', + `permission` varchar(255) DEFAULT NULL COMMENT '权限：执行器ID列表，多个逗号分割', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `i_username` (`username`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_user +-- ---------------------------- +INSERT INTO `xxl_job_user` VALUES ('1', 'admin', 'fea191a3fdd9f68503f9fa0e8d0293ab', '1', null); +INSERT INTO `xxl_job_user` VALUES ('2', 'query', '95497ea23bf8d27d86526717578e366f', '0', '3'); + +-- ---------------------------- +-- Table structure for `xxl_job_info` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_info`; +CREATE TABLE `xxl_job_info` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `job_group` int(11) NOT NULL COMMENT '执行器主键ID', + `job_cron` varchar(128) NOT NULL COMMENT '任务执行CRON', + `job_desc` varchar(255) NOT NULL, + `add_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `author` varchar(64) DEFAULT NULL COMMENT '作者', + `alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件', + `executor_route_strategy` varchar(50) DEFAULT NULL COMMENT '执行器路由策略', + `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', + `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', + `executor_block_strategy` varchar(50) DEFAULT NULL COMMENT '阻塞处理策略', + `executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT '任务执行超时时间，单位秒', + `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', + `glue_type` varchar(50) NOT NULL COMMENT 'GLUE类型', + `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', + `glue_remark` varchar(128) DEFAULT NULL COMMENT 'GLUE备注', + `glue_updatetime` datetime DEFAULT NULL COMMENT 'GLUE更新时间', + `child_jobid` varchar(255) DEFAULT NULL COMMENT '子任务ID，多个逗号分隔', + `trigger_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '调度状态：0-停止，1-运行', + `trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上次调度时间', + `trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '下次调度时间', + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=88 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + + +-- ---------------------------- +-- Records of xxl_job_group +-- ---------------------------- +INSERT INTO `xxl_job_group` VALUES (1, 'galaxy-executor', 'GDP', 1, '{{ job_executor_servers }}'); +INSERT INTO `xxl_job_group` VALUES (2, 'galaxy-executor', 'TSG_OLAP', 1, '{{ job_executor_servers }}'); +-- ---------------------------- +-- Records of xxl_job_info +-- ---------------------------- +INSERT INTO `xxl_job_info` VALUES (58, 1, '0 0 0 1/1 * ? ', 'set_druid_table_ttl', '2020-07-30 10:34:34', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteReportAndMetricsDataJobHandler', '{\"defaultMaxDays\":365,\"maxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 10:34:34', '', 1, 1704326400000, 1704412800000); +INSERT INTO `xxl_job_info` VALUES (59, 1, '0 0 0 1 * ?', 'clear_druid_table', '2020-07-30 11:32:08', '2022-04-12 02:38:39', 'zhq', '', 'FAILOVER', 'deleteAllReportAndMetricsDataJobHandler', '{\"maxDays\":88,\"defaultMaxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:32:08', '', 0, 0, 0); +INSERT INTO `xxl_job_info` VALUES (62, 1, '0 0 0 1/1 * ? ', 'set_hos_bucket_ttl', '2020-07-30 11:59:30', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteFilesJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:30', '', 1, 1704326400000, 1704412800000); +INSERT INTO `xxl_job_info` VALUES (63, 1, '0 0 0 1 * ?', 'clear_hos_file', '2020-07-30 11:59:43', '2023-10-26 10:43:46', 'zhq', '', 'FAILOVER', 'deleteAllFilesJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:43', '', 0, 0, 0); +INSERT INTO `xxl_job_info` VALUES (64, 1, '0 0 0 1 * ?', 'clear_clickhouse_table', '2020-07-30 13:47:25', '2022-04-12 02:38:39', 'zhq', '', 'FAILOVER', 'deleteAllTrafficDataJobHandler', '{\"maxDays\":77,\"defaultMaxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:25', '', 0, 0, 0); +INSERT INTO `xxl_job_info` VALUES (65, 1, '0 0 0 1/1 * ? ', 'set_clickhouse_table_ttl', '2020-07-30 13:47:38', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteTrafficDataJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:38', '', 1, 1704326400000, 1704412800000); + +INSERT INTO `xxl_job_info` VALUES (29, 2, '0 0 4 * * ?', 'get_clickhoue_storage_daily_usage', '2020-03-26 10:58:09', '2021-09-09 11:54:53', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE\",\"ckDayGrowth\":\"true\"}', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-03-26 10:58:09', '', 1, 1704254400000, 1704340800000); +INSERT INTO `xxl_job_info` VALUES (36, 2, '0 0/5 * * * ?', 'get_database_storage_quota', '2020-04-07 21:41:20', '2021-09-09 11:54:44', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE\",\"ckDayGrowth\":\"false\"}', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-04-07 21:41:20', '', 1, 1704338100000, 1704338400000); +INSERT INTO `xxl_job_info` VALUES (86, 2, '0 0 1 * * ?', 'run_druid_segment_compaction', '2021-07-21 14:13:57', '2023-12-12 02:21:05', 'xpf', '', 'FAILOVER', 'druidCompactSegmentJobHandler', '[\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_fqdns\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_domains\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_client_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_external_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_internal_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_subscribers\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"security_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"monitor_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"proxy_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"application_protocol_stat\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_general_stat\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_top_destination_ip_metrics_log\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"service_chaining_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"service_function_status\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"statistics_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_shaping_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"object_statistics\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"statistics_rule\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\",\r\n \"maxRowsInMemory\": 500000,\r\n \"maxBytesInMemory\": -1\r\n },\r\n \"context\":{\r\n \"druid.indexer.runner.javaOptsArray\": [\"-Xms4g\", \"-Xmx4g\", \"-XX:MaxDirectMemorySize=2g\"]\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"sys_storage_log\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n }\r\n]', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2021-07-21 14:13:57', '', 1, 1704330000000, 1704416400000); +INSERT INTO `xxl_job_info` VALUES (101, 2, '0 0 0 1/7 * ?', 'run_web_sketch_assessment', '2022-01-27 17:40:42', '2022-02-23 13:34:51', 'galaxy', '', 'FAILOVER', 'httpToStoreJobHandler', '[{\r\n \"url\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/sql/?option=long_term&resultId=1&query=SELECT server_domain AS name, arrayStringConcat(groupUniqArray(decoded_as),\',\') AS protocol, count(*) AS sessions FROM tsg_galaxy_v3.session_record WHERE recv_time >= toStartOfDay(now()- INTERVAL 7 DAY) AND recv_time < toStartOfDay(now()) AND notEmpty(server_domain) GROUP BY name ORDER BY sessions DESC LIMIT 500000;\",\r\n \"method\": \"get\",\r\n \"requestBody\": {},\r\n \"resultKey\": \"data\",\r\n \"store\": \"all\"\r\n}, {\r\n \"url\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/sql/?option=long_term&resultId=1&query=SELECT server_domain AS name, arrayStringConcat(groupUniqArray(decoded_as),\',\') AS protocol, count(*) AS sessions FROM tsg_galaxy_v3.session_record WHERE recv_time >= toStartOfDay(now()- INTERVAL 7 DAY) AND recv_time < toStartOfDay(now()) AND notEmpty(server_domain) AND empty(fqdn_category_list) GROUP BY name ORDER BY sessions DESC LIMIT 500000;\",\r\n \"method\": \"get\",\r\n \"requestBody\": {},\r\n \"resultKey\": \"data\",\r\n \"store\": \"uncategorized\"\r\n}]', 'COVER_EARLY', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2022-01-27 17:40:42', '', 1, 1704067200000, 1704672000000); +INSERT INTO `xxl_job_info` VALUES (141, 2, '0 0 2 * * ?', 'set_clickhouse_table_and_field_ttl', '2022-06-07 07:42:38', '2022-06-08 09:28:19', 'wk', '', 'FIRST', 'changeCkTtlJobHandler', '', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2022-06-07 07:42:38', '', 1, 1704333600000, 1704420000000); +INSERT INTO `xxl_job_info` VALUES (143, 2, '0 0 0/1 * * ?', 'run_ip_learning', '2022-11-09 10:29:20', '2023-12-26 04:01:24', 'galaxy', '', 'FIRST', '', '', 'SERIAL_EXECUTION', 0, 0, 'GLUE_SHELL', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/iplearning/ && ./start.sh\'', '0001', '2023-12-26 04:01:24', '', 1, 1704337200000, 1704340800000); +INSERT INTO `xxl_job_info` VALUES (145, 2, '0 0 1 * * ?', 'run_dos_baseline', '2022-11-10 06:15:27', '2023-11-13 05:37:41', 'galaxy', '', 'FIRST', '', '', 'SERIAL_EXECUTION', 0, 0, 'GLUE_SHELL', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/dos-baseline && ./start.sh\'', '0000', '2023-11-13 05:37:41', '', 1, 1704330000000, 1704416400000); +INSERT INTO `xxl_job_info` VALUES (155, 2, '0 0 16 1/7 * ?', 'run_geoip_to_mmdb', '2022-12-02 06:56:18', '2023-12-21 09:44:44', 'galaxy', '', 'FAILOVER', 'ipLocateLibraryJobHandler', '{\r\n \"command\": \"docker run --rm --network=host -v {{ deploy_dir }}/schedule-service/mmdb:/mmdb --name perl-mmdb perl:v3.1 sh -c \'cd /mmdb && perl mmdb-perl.pl\' \",\r\n \"path\": \"/opt/schedule-service/mmdb\",\r\n \"knowledgeBaseServer\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base\",\r\n \"knowledgeBaseList\": [{\"name\": \"ip_builtin\",\"builtin\":1 ,\"kb_id\": \"64af7077-eb9b-4b8f-80cf-2ceebc89bea9\" },\r\n {\"name\": \"ip_user_defined\",\"builtin\":0 ,\"kb_id\": \"004390bc-3135-4a6f-a492-3662ecb9e289\" }]\r\n}', 'DISCARD_LATER', 0, 0, 'BEAN', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/dos-baseline && ./start.sh\'', 'GLUE代码初始化', '2022-12-02 06:56:18', '', 0, 0, 0); +INSERT INTO `xxl_job_info` VALUES (158, 2, '0 0 1 * * ?', 'run_druid_segement_granularity_compact', '2023-02-20 05:59:59', '2023-05-11 03:39:14', 'galaxy', '', 'FAILOVER', 'druidMergeHistoryDataJobHandler', '[\r\n {\r\n \"dataSource\": \"service_function_status\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n },\r\n {\r\n \"dataSource\": \"service_chaining_rule_hits\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n },\r\n {\r\n \"dataSource\": \"traffic_shaping_rule_hits\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n }\r\n]', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2023-02-20 05:59:59', '', 1, 1704330000000, 1704416400000); + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 new file mode 100644 index 0000000..205a6a0 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 @@ -0,0 +1,23 @@ +#配置集的配置格式 +nacos.config.type=properties +#配置中心地址 +nacos.config.server-addr={{ vrrp_instance.default.virtual_ipaddress }}:8848 +#命名空间 +nacos.config.namespace={{ config_namespace }} +#数据集ID +nacos.config.data-id=galaxy-job-executor.properties +#开启自动刷新 +nacos.config.auto-refresh=true +#配置对应的分组 +nacos.config.group=Galaxy +#Nacos认证用户 +nacos.config.username=nacos +#Nacos认证密码 +nacos.config.password=nacos +#开启配置预加载功能 +nacos.config.bootstrap.enable=true +#开启Nacos支持日志级别的加载时机 +nacos.config.bootstrap.log-enable=true + +### log config +logging.config=./config/logback.xml diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 new file mode 100644 index 0000000..1aab54c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 @@ -0,0 +1,19 @@ +version: '3' +services: + galaxy-job-executor: + image: {{ image_name }}:{{ image_tag }} + container_name: galaxy-job-executor + environment: + JAVA_OPTS: "{{ galaxy_job_service.executor_java_opts }}" + ports: + - "8182:8182" + volumes: + - "{{ deploy_dir }}/{{ container_name }}/logs:/logs" + - "{{ deploy_dir }}/{{ container_name }}/data:/data" + - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-executor/config" + - "{{ deploy_dir }}/schedule-service:/opt/schedule-service" + - "/var/run/docker.sock:/var/run/docker.sock" + - "/usr/bin/docker:/usr/bin/docker" + restart: always + network_mode: "host" + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 new file mode 100644 index 0000000..5c8b0ef --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 @@ -0,0 +1,65 @@ +################################静态参数配置(修改后需要重启项目)################################ +### web port +server.port = 8185 +spring.application.name=galaxy-job-executor +### actuator +management.server.servlet.context-path=/actuator +management.health.mail.enabled=false +management.endpoints.web.exposure.include=* +#详细的应用健康信息 +management.endpoint.health.show-details=always +management.endpoint.metrics.enabled=true +management.endpoint.prometheus.enabled=true +management.metrics.export.prometheus.enabled=true +management.metrics.tags.application=${spring.application.name} +zookeeper.server={{ zookeeper_servers }} + +################################动态参数配置(修改后不需要重启项目)################################ +##存储配额文件服务器 +storge.files.hos-server={{ data_center_name }}|{{ vrrp_instance.oss.virtual_ipaddress }}:9098 +storge.files.token={{ hos_token }} +##存储配额查询druid +storge.analytic.server={{ data_center_name }}|{{ vrrp_instance.default.virtual_ipaddress }}:8089 +##存储配额查询clickhouse +storge.traffic.server={{ data_center_name }}|{{ vrrp_instance.default.virtual_ipaddress }}:8124 +storge.traffic.datasource=tsg_galaxy_v3 +storge.traffic.username=ENC(CNpgJlarVTNItkyW6DAQWA==) +storge.traffic.password=ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y) +#删除ttl +storge.traffic.system.parts=system.parts +#存储配额查询 +storge.traffic.system.partsclusters=system.parts_cluster +storge.traffic.system.disks=system.disks_cluster +storge.traffic.system.tables=system.tables_cluster +storge.traffic.system.clusters=system.clusters +#删除ttl白名单,多个逗号分隔 +storge.files.delete.exclusion=default,assessment_hos_bucket,knowledge_base_hos_bucket +storge.analytic.delete.exclusion=traffic_general_stat +storge.taffic.delete.exclusion=assessment_event_local +### xxl-job admin address list, such as "http://address" or "http://address01,http://address02" +xxl.job.admin.addresses={{ job_admin_servers }} +### xxl-job, access token +xxl.job.accessToken= +### xxl-job executor registry-address: default use address to registry , otherwise use ip:po +xxl.job.executor.appname=galaxy-executor +### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null +xxl.job.executor.address= +### xxl-job executor server-info +xxl.job.executor.ip= +xxl.job.executor.port=8886 +### xxl-job executor log-path +xxl.job.executor.logpath=/data/logs/jobhandler +### xxl-job executor log-retention-days +xxl.job.executor.logretentiondays=30 +## http pool config +### max connection number +http.pool.max.connection=500 +http.pool.request.timeout=120000 +http.pool.response.timeout=300000 +http.pool.max.per.route=300 +http.pool.connect.timeout=10000 +##指定kafka server的地址，集群配多个，中间，逗号隔开 +spring.kafka.bootstrap-servers={{ kafka_sink_servers }} +spring.kafka.ssl.enable=true +spring.kafka.ssl.username=ENC(RYN0it9o1vYynI1xlmX/Gw==) +spring.kafka.ssl.pin=ENC(vlb4F5Y3ff0Am/3MMPdIpYjZq8wcGNMd) diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 new file mode 100644 index 0000000..758d04d --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + ${LOG_PATTERN} + + + + ${LOG_PATH}/${LOG_FILE_NAME}.log + + ALL + + + ${LOG_PATTERN} + + + + ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz + + ${LOG_DAYS} + ${LOG_SIZE} + + + + + + + + + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 new file mode 100644 index 0000000..87689e5 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 @@ -0,0 +1,52 @@ +#!/bin/bash + +config_path_root=$1 +nacos_server={{ vrrp_instance.default.virtual_ipaddress }} + +tenant="{{ config_namespace }}" +app_name="galaxy-job-service" + +shell_result=1 + +# 推送配置 +function push() { + local config_file=$1 + local data_id="${config_file##*/}" + local suffix="${config_file##*.}" + + [ $suffix == "sql" ] && suffix="text" + [ $suffix == "yml" ] && suffix="yaml" + + local nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy" + local result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content@"$config_file") + if [[ $result -eq '200' ]]; then + sleep 2 + check=$(curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id") + #echo "$check $data_id" + + if [[ $check -ne '200' ]]; then + shell_result=0 + fi + fi +} + +# 获取本地文件 +function pushConfig() { + for config_path in $(readlink -f $config_path_root); do + if [ -d $config_path ]; then + for file in $(ls $config_path); do + config_path_file=$config_path/$file + if [ -f $config_path_file ]; then + push $config_path_file + fi + done + fi + if [ -f $config_path ]; then + push $config_path + fi + done +} + +pushConfig +echo $shell_result + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 new file mode 100644 index 0000000..bfd7c17 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash +set -e + +# 数据库连接参数 +upgrade_dir=${1:-`pwd`/upgrade-sql} + +# 指定当前版本和历史版本 +current_version="{{ solution_version }}" +old_version="{{ old_version }}" + +# 数据库连接参数 +db_hostname="{{ vrrp_instance.default.virtual_ipaddress }}" +db_name="xxl_job" +db_username="root" +db_password="{{ mariadb_default_pin }}" + +suffix=".sql" +# 获取更新目录中的所有SQL文件 +sql_files=$(find "${upgrade_dir}" -name "*$suffix") + +# 标记是否开始执行历史版本的标识 +start_execution=false + +# 循环处理每个SQL文件 +for file in ${sql_files}; do + # 从文件名中提取版本号 + filename=$(basename "$file") + version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.') + current_version="${current_version//./}" + old_version="${old_version//./}" + # 检查是否达到历史版本 + if [[ "${version}" -gt "${old_version}" ]]; then + start_execution=true + fi + + # 检查是否开始执行历史版本之后的SQL文件 + if [[ ${start_execution} = true ]]; then + echo "Executing SQL file: ${file}" + + # 执行SQL文件 + mysql -u "${db_username}" -p"${db_password}" -h "${db_hostname}" "${db_name}" < "${file}" + echo "Upgrade ${version} successfully" + fi + + # 检查是否达到当前版本 + if [[ "${version}" -ge "${current_version}" ]]; then + break + fi +done + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml new file mode 100644 index 0000000..de010b3 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml @@ -0,0 +1,26 @@ +software_packages_dir: "{{ role_path }}/../software-packages" + +admin_image_name: galaxy-job-admin +executor_image_name: galaxy-job-executor + +admin_image_tag: v1.3.220308 +executor_image_tag: v1.3.231220 + +admin_container_name: galaxy-job-admin +executor_container_name: galaxy-job-executor + + +mariadb_job_database: xxl_job +job_executor_servers: "{% for host in groups['galaxy_job_service'] %}http://{{ host }}:8886/{% if not loop.last %},{% endif %}{% endfor %}" + +job_admin_servers: "{% for host in groups['galaxy_job_service'] %}http://{{ host }}:8184/xxl-job-admin{% if not loop.last %},{% endif %}{% endfor %}" + +kafka_sink_servers: "{% for host in groups['kafka'] %}{{ host }}:9094{% if not loop.last %},{% endif %}{% endfor %}" +zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}" + +backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/" + +backup_items: + - config + - docker-compose.yml + - init diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/handlers/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/handlers/main.yml new file mode 100644 index 0000000..e6232c2 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/handlers/main.yml @@ -0,0 +1,42 @@ +- name: Stop Container + docker_container: + name: '{{ image_name }}' + state: absent + force_kill: true + +- name: Remove Image + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag }}' + state: absent + force_absent: true + +- name: Copy Image + copy: + src: 'files/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}' + force: yes + +- name: Unarchive Image + unarchive: + src: 'files/{{ image_name }}-{{ image_tag }}.tar.gz' + dest: '{{ deploy_dir }}/{{ container_name }}' + force: yes + +- name: Load Image + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag }}' + load_path: '{{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar' + source: load + force_tag: yes + force_source: yes + timeout: 300 + +- name: Start Container + docker_compose: + project_src: '{{ deploy_dir }}/{{ container_name }}' + +- name: Start Nginx + shell: 'docker restart galaxy-gateway-nginx' + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/backup.yml new file mode 100644 index 0000000..899461b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/backup.yml @@ -0,0 +1,25 @@ +- name: Check if backup directory exists + stat: + path: "{{ backup_path }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/{{ container_name }}/{{ item }}" + dest: "{{ backup_path }}" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}" + state: absent + when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/check.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/check.yml new file mode 100644 index 0000000..37029eb --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/check.yml @@ -0,0 +1,24 @@ +- name: Wait for galaxy-qgw-service port 8183 to become available + wait_for: + host: "{{ inventory_hostname }}" + port: 8183 + state: started + timeout: 120 + +- name: Request component status + uri: + url: "http://{{ inventory_hostname }}:8183/v1/monitor/component_status" + body_format: "json" + status_code: [200,503] + register: response_health + +- name: Checking component status + debug: + msg: "{{ response_health.json.components | dict2items | json_query(query) }}" + vars: + query: "[?value.status == 'DOWN']" + failed_when: response_health.json.status == "DOWN" + +- name: Checking metadata + uri: + url: "http://{{ inventory_hostname }}:8183/v1/troubleshooting/sanity?test=schema_consistency" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/deploy.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/deploy.yml new file mode 100644 index 0000000..90125a3 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/deploy.yml @@ -0,0 +1,47 @@ +- name: Creating config directory + file: + state: directory + path: "{{ deploy_dir }}/{{ container_name }}/config" + +- name: Copying config template + template: + src: "{{ item.src }}" + dest: "{{ deploy_dir }}/{{ container_name }}/{{ item.dest }}" + force: yes + loop: + - {src: 'docker-compose.yml.j2', dest: 'docker-compose.yml'} + - {src: 'application.yml.j2', dest: 'config/application.yml'} + - {src: 'hazelcast.yaml.j2', dest: 'config/hazelcast.yaml'} + notify: + - Start Container + +- name: Copying config files + copy: + src: "files/config/" + dest: "{{ deploy_dir }}/{{ container_name }}/config/" + force: yes + notify: + - Start Container + +- name: Copying image to {{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar + copy: + src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}/' + force: true + notify: + - Load Image + - Start Container + +- name: Copying nginx config and restart nginx + template: + src: "{{ container_name }}.conf.j2" + dest: "{{ deploy_dir }}/galaxy-gateway-nginx/conf/includes/{{ container_name }}.conf" + force: yes + notify: + - Start Nginx + delegate_to: "{{ item }}" + run_once: true + loop: "{{ groups.loadbalancer[:2] }}" + +- meta: flush_handlers + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/init.yml new file mode 100644 index 0000000..4c677ad --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/init.yml @@ -0,0 +1,60 @@ +- name: Check if directory exists + stat: + path: "{{ deploy_dir }}/{{ container_name }}/init" + register: directory_stat + ignore_errors: true + +- name: Remove files and directories + file: + path: "{{ deploy_dir }}/{{ container_name }}/init" + state: absent + when: directory_stat.stat.exists + +- name: Creating init directory + file: + state: directory + path: "{{ deploy_dir }}/{{ container_name }}/init" + +- name: Copying {{ container_name }} init template + template: + src: "{{ item.src }}.j2" + dest: "{{ deploy_dir }}/{{ container_name }}/init/{{ item.src }}" + mode: "{{ item.mode }}" + force: true + with_items: + - { src: "push_config.sh", mode: "0755" } + - { src: "galaxy-qgw-service.yml", mode: "0644" } + - { src: "create_mariadb_tables.sh", mode: "0755" } + +- name: Pushing config to nacos + shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && sh push_config.sh {{ item.conf }}' + register: post_result + until: post_result.stdout == '1' + retries: 3 + delay: 3 + failed_when: post_result.stdout != '1' + loop: + - { src: "{{ container_name }}" , conf: "galaxy-qgw-service.yml" } + run_once: true + +- name: Copying init files + copy: + src: "files/{{ item }}" + dest: "{{ deploy_dir }}/{{ container_name }}/init/" + force: yes + loop: + - 'data' + - 'schema' + - 'schema-upgrade-v3.1.jar' + +- name: Create database + shell: cd "{{ deploy_dir }}/{{ container_name }}/init"&& source /etc/profile && sh create_mariadb_tables.sh + run_once: true + +- name: Push schema to nacos + shell: cd "{{ deploy_dir }}/{{ container_name }}/init"&& source /etc/profile && java -jar schema-upgrade-v3.1.jar schema/ {{ vrrp_instance.default.virtual_ipaddress }} {{ config_namespace }} + run_once: true + +- name: Push schema to nacos + shell: cd "{{ deploy_dir }}/{{ container_name }}/init"&& source /etc/profile && sh push_config.sh galaxy-qgw-service.yml + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/main.yml new file mode 100644 index 0000000..80fea0f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/main.yml @@ -0,0 +1,17 @@ +- block: + - include: uninstall.yml + - include: init.yml + - include: deploy.yml + - include: check.yml + when: operation == 'install' + +- block: + - include: check.yml + - include: backup.yml + - include: upgrade.yml + - include: check.yml + when: operation == 'upgrade' + +- block: + - include: uninstall.yml + when: operation == 'uninstall' diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/uninstall.yml new file mode 100644 index 0000000..b623ddd --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/uninstall.yml @@ -0,0 +1,18 @@ +- name: Stopping and removing {{ container_name }} container + docker_container: + name: '{{ container_name }}' + state: absent + force_kill: true + +- name: Removing old {{ image_name }} image + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag }}' + state: absent + force_absent: true + +- name: Ansible delete old {{ deploy_dir }}/{{ container_name }} + file: + path: '{{ deploy_dir }}/{{ container_name }}' + state: absent + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/upgrade.yml new file mode 100644 index 0000000..c190537 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/upgrade.yml @@ -0,0 +1,95 @@ +- name: Check if directory exists + stat: + path: "{{ deploy_dir }}/{{ container_name }}/init" + register: directory_stat + ignore_errors: true + +- name: Remove files and directories + file: + path: "{{ deploy_dir }}/{{ container_name }}/init" + state: absent + when: directory_stat.stat.exists + +- name: Creating init directory + file: + state: directory + path: "{{ deploy_dir }}/{{ container_name }}/init" + +- name: Copying {{ container_name }} init template + template: + src: "{{ item.src }}" + dest: "{{ deploy_dir }}/{{ container_name }}/init/{{ item.dest }}" + mode: 0755 + force: true + with_items: + - { src: "push_config.sh.j2", dest: "push_config.sh" } + - { src: "create_mariadb_tables.sh.j2", dest: "create_mariadb_tables.sh" } + - { src: "merge-config.sh.j2", dest: "merge-config.sh" } + - { src: 'keys.txt', dest: 'keys.txt'} + +- name: Copying init files + copy: + src: "files/{{ item }}" + dest: "{{ deploy_dir }}/{{ container_name }}/init/" + force: yes + loop: + - 'data' + - 'schema' + - 'schema-upgrade-v3.1.jar' + +- name: Copying config files + copy: + src: "files/config/" + dest: "{{ deploy_dir }}/{{ container_name }}/config/" + force: yes + notify: + - Start Container + +- name: Copying config template + template: + src: "{{ item.src }}" + dest: "{{ deploy_dir }}/{{ container_name }}/{{ item.dest }}" + force: yes + loop: + - {src: 'docker-compose.yml.j2', dest: 'init/docker-compose.yml.j2'} + - {src: "galaxy-qgw-service.yml.j2", dest: "init/galaxy-qgw-service.yml.j2"} + - {src: 'application.yml.j2', dest: 'config/application.yml'} + - {src: 'hazelcast.yaml.j2', dest: 'config/hazelcast.yaml'} + notify: + - Start Container + +- name: Merge online config + shell: cd {{ deploy_dir }}/{{ container_name }}/init && sh merge-config.sh galaxy-qgw-service.yml galaxy-qgw-service.yml.j2 nacos + +- name: Merge online docker-compose.yml + shell: cd {{ deploy_dir }}/{{ container_name }}/init && sh merge-config.sh {{ deploy_dir }}/{{ container_name }}/docker-compose.yml docker-compose.yml.j2 + +- name: Create database + shell: cd {{ deploy_dir }}/{{ container_name }}/init && source /etc/profile && sh create_mariadb_tables.sh + run_once: true + +- name: Push schema to nacos + shell: cd {{ deploy_dir }}/{{ container_name }}/init && source /etc/profile && java -jar schema-upgrade-v3.1.jar schema/ {{ vrrp_instance.default.virtual_ipaddress }} {{ config_namespace }} + run_once: true + +- name: Push config to nacos + shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && source /etc/profile && sh push_config.sh {{ item.conf }}' + register: post_result + until: post_result.stdout == '1' + retries: 3 + delay: 3 + failed_when: post_result.stdout != '1' + loop: + - { src: "{{ container_name }}" , conf: "galaxy-qgw-service.yml" } + run_once: true + +- name: Copying image to {{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar + copy: + src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}/' + force: true + notify: + - Load Image + - Start Container + +- meta: flush_handlers diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/application.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/application.yml.j2 new file mode 100644 index 0000000..5c103a5 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/application.yml.j2 @@ -0,0 +1,22 @@ +nacos: + config: + type: yaml #配置集的配置格式 + server-addr: {{ vrrp_instance.default.virtual_ipaddress }}:8848 #配置中心地址 + namespace: {{ config_namespace }} #命名空间 + data-id: galaxy-qgw-service.yml # 数据集ID + auto-refresh: true #开启自动刷新 + group: Galaxy #配置对应的分组 + username: nacos #Nacos认证用户 + password: nacos + bootstrap: + enable: true #开启配置预加载功能 + log: + enable: true #开启Nacos支持日志级别的加载时机 +spring: + servlet: + multipart: + max-file-size: 5368709120 + max-request-size: 5368709120 +logging: + config: ./config/log4j2.xml + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/create_mariadb_tables.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/create_mariadb_tables.sh.j2 new file mode 100644 index 0000000..46c72fa --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/create_mariadb_tables.sh.j2 @@ -0,0 +1,4 @@ +#!/bin/bash + +mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ inventory_hostname }} -e "create database if not exists {{ mariadb_common_database }} default character set utf8mb4 collate utf8mb4_general_ci;" + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..765b4c5 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/docker-compose.yml.j2 @@ -0,0 +1,14 @@ +version: "3" +services: + galaxy-qgw-service: + image: {{ image_name }}:{{ image_tag }} + environment: + JAVA_OPTS: "{{ galaxy_qgw_service.java_opts }}" + container_name: {{ container_name }} + ports: + - "8183:8183" + volumes: + - "{{ deploy_dir }}/{{ container_name }}/logs:/home/tsg/galaxy/galaxy-qgw-service/logs" + - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-qgw-service/config" + restart: always + network_mode: "host" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.conf.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.conf.j2 new file mode 100644 index 0000000..9f90ded --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.conf.j2 @@ -0,0 +1,24 @@ +upstream qgwService { + +{% for host in groups['galaxy_qgw_service'] %} + server {{ host }}:8183; +{% endfor %} + +} + +server { + listen 9999; + server_name localhost; + + location / { + proxy_pass http://qgwService; #请求转发到查询引擎集群 + proxy_http_version 1.1; #指定使用http1.1版本 + proxy_read_timeout 21600; #等待后端服务响应的最大时长 + gzip on; #开启压缩 + gzip_comp_level 6; #压缩级别 + gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩 + gzip_types application/json; #压缩文件类型 + gzip_vary on; #是否传输gzip压缩标志 + } +} + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2 new file mode 100644 index 0000000..4941b2d --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2 @@ -0,0 +1,175 @@ +##############静态参数配置(修改后需要重启项目)############## +## 服务配置 +server: + port: 8183 +## 监控参数 +management: + metrics: + tags: + application: ${project.name} + endpoint: + health: + show-details: always + shutdown: + enabled: true + health: + redis: + enabled: false + db: + enabled: false + endpoints: + web: + exposure: + include: '*' + exclude: env,auditevents,beans,conditions,info + base-path: /v1/monitor + path-mapping: + health: /component_status +#MySql configuration +spring: + sleuth: + enabled: true + traceId128: true + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://{{ vrrp_instance.default.virtual_ipaddress }}:3306/tsg_olap?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=UTC&failOverReadOnly=false&connectTimeout=10000&socketTimeout=30000 + username: ENC(63aTpwv2vH0vPikW+3Jjig==) + password: ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y) + flyway: + enabled: true # 是否开启 + encoding: UTF-8 # 编码 + sql-migration-prefix: V # 脚本文件的前缀，默认为V + sql-migration-separator: __ # 双下划线 + baseline-on-migrate: true # 连接数据库中存在表时设置为true + check-location: false + locations: filesystem:./config/flyway # 脚本路径 + clean-disabled: true # flyway 的 clean 命令会删除指定 schema 下的所有 table, 生产务必禁掉。这个默认值是 false 理论上作为默认配置是不科学的 + validate-on-migrate: true # 执行迁移时是否自动调用验证 当你的 版本不符合逻辑 比如 你先执行了 DML 而没有 对应的DDL 会抛出异常 + placeholder-replacement: false # 不做取值替换 默认替换为 ${} ，初始化sql中有sql语句存在freemarker替换，所以禁用此项 +## Hbase configuration +hbase: + zookeeperQuorum: {{ zookeeper_noport }} + zookeeperZnodeParent: /hbase + rpcTimeout: 60000 + report: + dbname: tsg_galaxy + tableName: saved_query_result + columnFamily: response + columnName: result +##############动态参数配置(修改后不需要重启项目)############## +## 项目参数 +project: + name: galaxy-qgw-service-nacos + description: 统一数据查询网关 + version: 1.1 + groupId: com.mesalab + artifactId: galaxy-qgw-service + basedir: + corePackage: com.mesalab.common + servicePackage: com.mesalab.qgw + author: + name: darnell + url: + email: +## ClickhHouse configuration +clickhouse: + url: http://{{ vrrp_instance.default.virtual_ipaddress }}:8123 + dbname: tsg_galaxy_v3 + enableApproximateOptimizer: true + realTimeAccount: + username: ENC(hYFEuRJJWd93TZg5VbK/o3OXUmxI/irv) + pin: ENC(qUA355VopKSx6kwwwXZwqWWEYSu76Slz) + socketTimeOut: 120000 + longTermAccount: + username: ENC(z3gVlaa7Q4IMDqtUgvhwoJzKAkH1Uznc) + pin: ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y) + socketTimeOut: 21700000 +## Druid configuration +druid: + url: {{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/v2/sql + dbname: druid + socketTimeOut: 120000 +## xxl-job-admin configuration +xxl-job-admin: + url: http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin + userName: ENC(v8NKKlLWitI7vGhuGYorRQ==) + pin: ENC(xTCafSByYA6SyvhUJ6vrh2VSXuZAVK/O) +## ArangoDB configuration +arango: + server: http://{{ groups.arangodb[0] }}:8529 + database: tsg_galaxy_v3 + username: ENC(63aTpwv2vH0vPikW+3Jjig==) + pin: ENC(PBdMaxfC3u+HMzjjij2tyuJWeooSuZNW) + jwturl: ${arango.server}/_db/${arango.database}/_open/auth + queryurl: ${arango.server}/_db/${arango.database}/_api/cursor + maxrows: 10000 + socketTimeOut: 300000 +## hos cfg +hos: + uri: http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos + bucket: knowledge_base_bucket + token: {{ hos_token }} + multiUploadStartSize: 5368709120 + multiUploadPartSize: 10485760 +## Engine、Service configuration +engine: + maxCacheNum: 1048575 + defaultResultNum: 100000 + sampleDataPeriod: 1800 +service: + entity: + activeClientIP: 10000 + topServerIP: + TCPBySession: 100000 + TCPByUniqClientIP: 100000 + UDPBySession: 100000 + UDPByUniqClientIP: 40000 + topSNI: + defaultSize: 30000 + dataset: 2000000 + gtpc: + defaultSize: 100000 +## knowledge value 格式 name:版本 +knowledge: + corn: 0 0 */1 * * ? + ipLocation: + userDefined: 004390bc-3135-4a6f-a492-3662ecb9e289_latest + builtIn: 64af7077-eb9b-4b8f-80cf-2ceebc89bea9_latest + asn: f9f6bc91-2142-4673-8249-e097c00fe1ea_latest +## http pool config +http: + pool: + connect: + timeout: 30000 + max: + connection: 500 + per: + route: 200 + request: + timeout: 30000 + response: + timeout: 120000 +## job cfg +job: + timeSlicing: + enabled: true + interval: 1800 #单位: Second + longPolling: + enabled: true + interactive: + timeout: 10000 + response: + timeout: 3000 + execution: + timeout: 300000 + pool: + corePoolSize: 20 + maxPoolSize: 30 + queueCapacity: 10 +## task cfg +task: + pool: + corePoolSize: 8 + maxPoolSize: 16 + queueCapacity: 1000 + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/hazelcast.yaml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/hazelcast.yaml.j2 new file mode 100644 index 0000000..5d5e99e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/hazelcast.yaml.j2 @@ -0,0 +1,16 @@ +hazelcast: + cluster-name: qgw-cluster + network: + join: + multicast: + enabled: false + tcp-ip: + enabled: true + member-list: +{% for host in groups['galaxy_qgw_service'] %} + - {{ host }} +{% endfor %} + map: + adHocCache: + time-to-live-seconds: 120 + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/keys.txt b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/keys.txt new file mode 100644 index 0000000..0784ff4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/keys.txt @@ -0,0 +1 @@ +services.galaxy-qgw-service.environment.JAVA_OPTS diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/merge-config.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/merge-config.sh.j2 new file mode 100644 index 0000000..aba794c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/merge-config.sh.j2 @@ -0,0 +1,83 @@ +#!/bin/bash +set -e + +host="{{ vrrp_instance.default.virtual_ipaddress }}" +tenant="{{ config_namespace }}" +group_id="Galaxy" +app_name="" +username="nacos" +password="nacos" + +online_file=$1 +output_file=$1 +template_file=$2 + +online_file=$1 +output_file=$1 +template_file=$2 + +[[ -z $1 || -z $2 ]] && echo "Missing input file !" && exit 1 + +function format_file { + local file="$1" + echo "format "$file + if [ -f "$file" ]; then + cat "$file" | tr -d "\r" >"$file"_new + mv "$file"_new $file + fi +} + +function download_nacos_config() { + local data_id=$online_file + + config_url="http://$host:8848/nacos/v1/cs/configs?dataId=$data_id&group=$group_id&appName=$app_name&username=$username&password=$password&tenant=$tenant" + online_file=nacos-"$data_id" + curl -s -o $online_file $config_url + format_file $online_file +} + +function merge_yml_config() { + grep -v '^$' keys.txt | while IFS= read -r key; do + # 使用yq命令读取YAML文件中的键值 + value=$(yq eval ".${key}" "$online_file") + if [[ -n $value && "$value" != "null" ]]; then + + yq eval ".${key} = \"${value}\"" -i $template_file + echo "upgrade "$(yq eval ".${key}" $template_file) + fi + + done + rm -r "$online_file" + mv $template_file $output_file + echo -e "\033[32mmerge success\033[0m" + +} + +function merge_properties_config() { + grep -v '^$' keys.txt | while IFS= read -r key; do + value=$(grep -E "^[[:blank:]]*${key}[[:blank:]]*=" "$online_file" | cut -d'=' -f1 --complement) + if [[ -n $value && "$value" != "null" ]]; then + sed -i "s/^[[:blank:]]*${key}[[:blank:]]*=\(.*\)$/${key}=$value/" $template_file + echo "upgrade "$(grep -E "^[[:blank:]]*${key}[[:blank:]]*=" "$template_file") + fi + + done + rm -r "$online_file" + mv $template_file $output_file + echo -e "\033[32mmerge success\033[0m" + +} + +[[ $3 == 'nacos' ]] && download_nacos_config + +suffix="${online_file##*.}" + +case $suffix in +yaml | yml) + merge_yml_config + ;; +properties) + merge_properties_config + ;; +esac + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/push_config.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/push_config.sh.j2 new file mode 100644 index 0000000..a5ea43f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/push_config.sh.j2 @@ -0,0 +1,52 @@ +#!/bin/bash + +config_path_root=$1 +nacos_server={{ vrrp_instance.default.virtual_ipaddress }} + +tenant="{{ config_namespace }}" +app_name="{{ container_name }}" + +shell_result=1 + +# 推送配置 +function push() { + local config_file=$1 + local data_id="${config_file##*/}" + local suffix="${config_file##*.}" + + [ $suffix == "sql" ] && suffix="text" + [ $suffix == "yml" ] && suffix="yaml" + + local nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy" + local result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content@"$config_file") + if [[ $result -eq '200' ]]; then + sleep 2 + check=$(curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id") + #echo "$check $data_id" + + if [[ $check -ne '200' ]]; then + shell_result=0 + fi + fi +} + +# 获取本地文件 +function pushConfig() { + for config_path in $(readlink -f $config_path_root); do + if [ -d $config_path ]; then + for file in $(ls $config_path); do + config_path_file=$config_path/$file + if [ -f $config_path_file ]; then + push $config_path_file + fi + done + fi + if [ -f $config_path ]; then + push $config_path + fi + done +} + +pushConfig +echo $shell_result + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/upload_knowlodge.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/upload_knowlodge.sh.j2 new file mode 100644 index 0000000..b82e0d0 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/upload_knowlodge.sh.j2 @@ -0,0 +1,87 @@ +#!/bin/bash + +COMMON_PATH={{ deploy_dir }}/{{ qgw_soft_home_path }} + +function push_knowledge_file(){ +FILE_NAME=$1 +FILE_PATH=$2 +TYPE=$3 +FORMAT=$4 + +curl_result=$(curl -s --retry-delay 3 --retry 3 --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/knowledge_base/v1' --form 'file=@"'$FILE_PATH'/'$FILE_NAME'.'$FORMAT'"' --form 'name='$FILE_NAME'' --form 'format='$FORMAT'' --form 'type='$TYPE'' | jq '.status') + +if [[ $curl_result -ne '200' ]];then + echo 0 + echo "The $FILE_NAME.$FORMAT upload to the knowledge base failed." + exit 1 +fi +} + +function push_knowledge_csv(){ +ID=$1 +NAME=$2 +TYPE=$3 + +curl_result=$(curl -s --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/knowledge_base/v1' \ +--form 'file=@"./data/'$NAME'.csv"' --form 'name='$NAME'' --form 'format=csv' --form 'type='$TYPE'' --form 'id='$ID'' | jq '.status') + +if [[ $curl_result -ne '200' ]];then + echo 0 + echo "The $FILE_NAME.$FORMAT upload to the knowledge base failed." + exit 1 +fi +} + +function push_knowledge_mmdb(){ +ID=$1 +NAME=$2 +TYPE=$3 + +curl_result=$(curl -s --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/knowledge_base/v1' \ +--form 'file=@"./data/'$TYPE'.mmdb"' --form 'name='$NAME'' --form 'format=mmdb' --form 'type='$TYPE'' --form 'id='$ID'' | jq '.status') + +if [[ $curl_result -ne '200' ]];then + echo 0 + echo "The $FILE_NAME.$FORMAT upload to the knowledge base failed." + exit 1 +fi +} + +function pushConfig(){ + +{% if project == "tsg" %} +push_knowledge_file asn_v4 $COMMON_PATH/data asn mmdb +push_knowledge_file asn_v6 $COMMON_PATH/data asn mmdb +push_knowledge_file ip_v4_built_in $COMMON_PATH/data ip_location mmdb +push_knowledge_file ip_v4_user_defined $COMMON_PATH/data ip_location mmdb +push_knowledge_file ip_v6_built_in $COMMON_PATH/data ip_location mmdb +push_knowledge_file ip_v6_user_defined $COMMON_PATH/data ip_location mmdb +{% endif %} + + +{% if project == "cn" %} +push_knowledge_csv 3 cn_dns_server_info_built_in cn_dns_server_info_built_in +push_knowledge_csv 4 cn_fqdn_icp_built_in cn_fqdn_icp_built_in +push_knowledge_csv 5 cn_fqdn_category_built_in cn_fqdn_category_built_in +push_knowledge_csv 6 cn_fqdn_who_is_built_in cn_fqdn_who_is_built_in +push_knowledge_csv 7 cn_ioc_malware cn_ioc_malware +push_knowledge_csv 8 cn_ioc_darkweb cn_ioc_darkweb +push_knowledge_csv 9 cn_app_category_built_in cn_app_category_built_in +push_knowledge_csv 10 cn_psiphon3_ip cn_psiphon3_ip +push_knowledge_csv 11 cn_idc_renter_built_in cn_idc_renter_built_in +push_knowledge_csv 12 cn_internal_ip_built_in cn_internal_ip_built_in +push_knowledge_csv 13 cn_link_direction_built_in cn_link_direction_built_in + + +push_knowledge_mmdb 16266e3e2aff061d cn_ip_location_built_in cn_ipv4_location_built_in +push_knowledge_mmdb f8e1f4021a069b7d cn_ip_location_built_in cn_ipv6_location_built_in +push_knowledge_mmdb 15a614c6f12f1ad5 cn_ip_asn_built_in cn_ipv4_asn_built_in +push_knowledge_mmdb f4b2ed071a57b35c cn_ip_asn_built_in cn_ipv6_asn_built_in + +{% endif %} + +} + +pushConfig + +echo 1 diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/vars/main.yml new file mode 100644 index 0000000..1e9b201 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/vars/main.yml @@ -0,0 +1,16 @@ +software_packages_dir: "{{ role_path }}/../software-packages" + +image_name: galaxy-qgw-service +image_tag: 374 +container_name: galaxy-qgw-service + +backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/{{ container_name }}" + +backup_items: + - config + - docker-compose.yml + - init + + +mariadb_common_database: tsg_olap +zookeeper_noport: "{% for host in groups['zookeeper'] %}{{ host }}{% if not loop.last %},{% endif %}{% endfor %}:2181" diff --git a/tsg-olap/parcels/roles/initialization/hbase/tasks/init.yml b/tsg-olap/parcels/roles/initialization/hbase/tasks/init.yml new file mode 100644 index 0000000..e016d42 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/hbase/tasks/init.yml @@ -0,0 +1,54 @@ +- name: Setting Project variable + shell: cat /etc/hostname + register: hostname_master_out + run_once: true + delegate_to: "{{ groups.hbase[0] }}" + +- name: Setting Project variable + shell: cat /etc/hostname + register: hostname_slave_out + run_once: true + delegate_to: "{{ groups.hbase[1] if groups['hbase']|length > 1 else groups.hbase[0] }}" + +- name: project to ansible variable + set_fact: + hostname_master: "{{hostname_master_out.stdout}}" + hostname_slave: "{{hostname_slave_out.stdout}}" + init_path: "{{ deploy_dir }}/hbase/init" + container_name: 'HMaster' + +- name: Create {{ init_path }} + file: + path: '{{ init_path }}' + state: directory + +- name: Copying HBase initshell files to {{ init_path }} + template: + src: '{{ item }}' + dest: '{{ init_path }}/{{ item }}' + mode: 0755 + force: true + loop: + - create-hbase-table.sh + - create-phoenix-table.sh + - rsgroup.sh + +- name: Creating hbase table + shell: cd {{ init_path }} && sh create-hbase-table.sh | grep ERROR | grep -v "already exists" + register: result + failed_when: "'ERROR' in result.stdout" + run_once: true + +- name: Creating phoenix table + shell: cd {{ init_path }} && sh create-phoenix-table.sh | grep ERROR | grep -v "already exists" + register: result + failed_when: "'ERROR' in result.stdout" + run_once: true + +- name: Enable RsGroup + shell: cd {{ init_path }} && sh rsgroup.sh | grep ERROR | egrep -v "already exists|Target RSGroup important is same as source|Source RSGroup important is same as target" + register: result + failed_when: "'ERROR' in result.stdout" + when: groups.hbase | length > 1 + run_once: true + diff --git a/tsg-olap/parcels/roles/initialization/hbase/tasks/main.yml b/tsg-olap/parcels/roles/initialization/hbase/tasks/main.yml new file mode 100644 index 0000000..125d374 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/hbase/tasks/main.yml @@ -0,0 +1,5 @@ +- include: init.yml + when: operation == 'install' + +- include: upgrade.yml + when: operation == 'upgrade' diff --git a/tsg-olap/parcels/roles/initialization/hbase/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/hbase/tasks/upgrade.yml new file mode 100644 index 0000000..3b2304b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/hbase/tasks/upgrade.yml @@ -0,0 +1,41 @@ +- block: + - name: project to ansible variable + set_fact: + upgrade_path: "{{ deploy_dir }}/hbase/upgrade" + container_name: 'HMaster' + + - name: Clean {{ upgrade_path }} + file: + path: '{{ upgrade_path }}' + state: absent + + - name: Create {{ upgrade_path }} + file: + path: '{{ upgrade_path }}' + state: directory + + - name: Copying Upgrade files to {{ upgrade_path }} + copy: + src: "files/upgrade/" + dest: "{{ upgrade_path }}/" + force: true + + - name: Copying HBase upgrade template to {{ upgrade_path }} + template: + src: '{{ item }}' + dest: '{{ upgrade_path }}/{{ item }}' + mode: 0755 + force: true + loop: + - upgrade-hbase-data.sh + - upgrade-phoenix-data.sh + + - name: Upgrade hbase table data + shell: cd {{ upgrade_path }} && sh upgrade-hbase-data.sh + run_once: true + + - name: Upgrade phoenix table data + shell: cd {{ upgrade_path }} && sh upgrade-phoenix-data.sh + run_once: true + + when: inventory_hostname in groups['hbase'][:2] diff --git a/tsg-olap/parcels/roles/initialization/hbase/templates/create-hbase-table.sh b/tsg-olap/parcels/roles/initialization/hbase/templates/create-hbase-table.sh new file mode 100644 index 0000000..6dfe60f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/hbase/templates/create-hbase-table.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +source /etc/profile + +docker exec -i {{ container_name }} hbase shell < 'response', VERSIONS => 1,COMPRESSION => 'GZ',IS_MOB => true, MOB_THRESHOLD => 0}, {NAME => 'detail',COMPRESSION => 'GZ',VERSIONS => 1} + +create 'dos:ddos_traffic_baselines', 'TCP SYN Flood','UDP Flood','ICMP Flood','DNS Flood' + + +EOF diff --git a/tsg-olap/parcels/roles/initialization/hbase/templates/create-phoenix-table.sh b/tsg-olap/parcels/roles/initialization/hbase/templates/create-phoenix-table.sh new file mode 100644 index 0000000..2fe23d5 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/hbase/templates/create-phoenix-table.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +source /etc/profile + +phoenix_path={{ deploy_dir }}/hbase/phoenix-hbase-2.2-5.1.2-bin/bin + +$phoenix_path/sqlline.py<1 else 1 }} +kafka_servers="{% for host in groups['kafka'] %}{{ host }}:9092{% if not loop.last %},{% endif %}{% endfor %}" + +common_topic=( + BGP-RECORD + DOS-EVENT + DOS-SKETCH-RECORD + NETWORK-TRAFFIC-METRIC + OBJECT-STATISTICS-METRIC + POLICY-RULE-METRIC + PROXY-EVENT + PROXY-EVENT-PROCESSED + PXY-EXCH-INTERMEDIA-CERT + SESSION-RECORD-IPFIX + STATISTICS-RULE-METRIC + SYS-STORAGE + TRAFFIC-FILE-STREAM-RECORD + TRAFFIC-TOP-DESTINATION-IP-METRIC + TRAFFIC-TOP-METRIC + TROUBLESHOOTING-FILE-STREAM-RECORD + VOIP-CONVERSATION-RECORD + VOIP-RECORD + VOIP-RECORD-PROCESSED +) + +multi_partion_topic=( + TRANSACTION-RECORD + TRANSACTION-RECORD-PROCESSED + SESSION-RECORD + SESSION-RECORD-PROCESSED +) + +function getTopic() { + docker exec kafka kafka-topics.sh --bootstrap-server $kafka_servers --list +} + +function createTopic() { + exist_topic=$(echo "$exist_topic_list" | grep -wx "$3" | wc -l) + if [ $exist_topic -eq '0' ]; then + docker exec kafka kafka-topics.sh '--create' '--bootstrap-server' "$kafka_servers" '--replication-factor' $1 '--partitions' $2 '--topic' $3 + fi +} + +function checkTopic() { + exist_topic_list=$(getTopic) + all_topic=("${common_topic[@]}" "${multi_partion_topic[@]}") + for topic_name in ${all_topic[@]}; do + exist_topic=$(echo "$exist_topic_list" | grep -wx "$topic_name" | wc -l) + if [ $exist_topic -eq '0' ]; then + echo "Topic $topic_name creation failure" + exit 1 + fi + done + +} + +exist_topic_list=$(getTopic) +for topic_name in ${common_topic[@]}; do + createTopic $replication_nums $partion_nums $topic_name +done + +for topic_name in ${multi_partion_topic[@]}; do + createTopic 1 $((partion_nums * 3)) $topic_name +done + +checkTopic + diff --git a/tsg-olap/parcels/roles/initialization/kafka/vars/main.yml b/tsg-olap/parcels/roles/initialization/kafka/vars/main.yml new file mode 100644 index 0000000..d9b473f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/kafka/vars/main.yml @@ -0,0 +1 @@ +kafka_servers: "{% for host in groups['kafka'] %}{{ host }}:9092{% if not loop.last %},{% endif %}{% endfor %}" diff --git a/tsg-olap/parcels/roles/initialization/nacos/tasks/main.yml b/tsg-olap/parcels/roles/initialization/nacos/tasks/main.yml new file mode 100644 index 0000000..1131a78 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/nacos/tasks/main.yml @@ -0,0 +1,14 @@ +- name: Create Nacos Namespace + uri: + url: "http://{{ vrrp_instance.default.virtual_ipaddress }}:8848/nacos/v1/console/namespaces?username=nacos&password={{ nacos_default_pin }}" + method: POST + body_format: form-urlencoded + body: + namespaceId: '' + namespaceName: "{{ config_namespace }}" + namespaceDesc: "{{ config_namespace }}" + customNamespaceId: "{{ config_namespace }}" + retries: 3 + delay: 3 + run_once: true + when: operation == 'install' diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/handlers/main.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/handlers/main.yml new file mode 100644 index 0000000..5787af9 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/handlers/main.yml @@ -0,0 +1,43 @@ +- name: Stop Container + docker_container: + name: '{{ image_name }}' + state: absent + force_kill: true + +- name: Remove Image + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag }}' + state: absent + force_absent: true + +- name: Copy Image + copy: + src: 'files/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}' + force: yes + +- name: Unarchive Image + unarchive: + src: 'files/{{ image_name }}-{{ image_tag }}.tar.gz' + dest: '{{ deploy_dir }}/{{ container_name }}' + force: yes + +- name: Load Image + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag }}' + load_path: '{{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar' + source: load + force_tag: yes + force_source: yes + timeout: 300 + +- name: Start Container + docker_compose: + project_src: '{{ deploy_dir }}/{{ container_name }}' + +- name: Start Nginx + docker_container: + name: 'galaxy-gateway-nginx' + state: started diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/backup.yml new file mode 100644 index 0000000..899461b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/backup.yml @@ -0,0 +1,25 @@ +- name: Check if backup directory exists + stat: + path: "{{ backup_path }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/{{ container_name }}/{{ item }}" + dest: "{{ backup_path }}" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}" + state: absent + when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/check.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/check.yml new file mode 100644 index 0000000..a920d26 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/check.yml @@ -0,0 +1,23 @@ +- name: Wait for {{ container_name }} port 9093 to become available + wait_for: + host: "{{ inventory_hostname }}" + port: 9093 + state: started + timeout: 120 + +- name: Request component status + uri: + url: "http://{{ inventory_hostname }}:9093/monitor" + body_format: "json" + status_code: [200] + return_content: true + register: response_health + +- name: parse json + set_fact: + content_json: "{{ response_health.content | from_json }}" + +- name: Checking status message + debug: + msg: "{{ response_health.content }}" + failed_when: content_json.status != "active" and content_json.status != "standby" diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/deploy.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/deploy.yml new file mode 100644 index 0000000..c253e7e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/deploy.yml @@ -0,0 +1,29 @@ +- name: Creating directory + file: + state: directory + path: '{{ deploy_dir }}/{{ container_name }}/config' + +- name: Copying config + template: + src: "{{ item.src }}" + dest: '{{ deploy_dir }}/{{ container_name }}/{{ item.dest }}' + force: yes + loop: + - { src: 'application.yml.j2', dest: 'config/application.yml' } + - { src: 'log4j2-dev.xml.j2', dest: 'config/log4j2-dev.xml' } + - { src: 'docker-compose.yml.j2', dest: 'docker-compose.yml' } + notify: + - Start Container + +- name: Copying image to {{ deploy_dir }}/{{ container_name }} + copy: + src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}/' + force: true + notify: + - Load Image + - Start Container + + +- meta: flush_handlers + diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/init.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/init.yml new file mode 100644 index 0000000..530e18e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/init.yml @@ -0,0 +1,25 @@ +- name: Creating directory + file: + state: directory + path: '{{ deploy_dir }}/{{ container_name }}/init' + +- name: Copying init files + template: + src: "{{ item.src }}.j2" + dest: "{{ deploy_dir }}/{{ container_name }}/init/{{ item.src }}" + mode: "{{ item.mode }}" + force: true + loop: + - { src: "push_config.sh", mode: "0755" } + - { src: "saved-query-scheduler.yml", mode: "0644" } + +- name: Pushing config to nacos + shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && sh push_config.sh {{ item.conf }}' + register: post_result + until: post_result.stdout == '1' + retries: 3 + delay: 3 + failed_when: post_result.stdout != '1' + loop: + - { src: "{{ container_name }}" , conf: "saved-query-scheduler.yml" } + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/main.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/main.yml new file mode 100644 index 0000000..7f03439 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/main.yml @@ -0,0 +1,19 @@ +- block: + - include: uninstall.yml + - include: init.yml + - include: deploy.yml + - include: check.yml + when: operation == 'install' + +- block: + - include: check.yml + - include: backup.yml + - include: upgrade.yml + - include: init.yml + - include: deploy.yml + - include: check.yml + when: operation == 'upgrade' + +- block: + - include: uninstall.yml + when: operation == 'uninstall' diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/uninstall.yml new file mode 100644 index 0000000..b623ddd --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/uninstall.yml @@ -0,0 +1,18 @@ +- name: Stopping and removing {{ container_name }} container + docker_container: + name: '{{ container_name }}' + state: absent + force_kill: true + +- name: Removing old {{ image_name }} image + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag }}' + state: absent + force_absent: true + +- name: Ansible delete old {{ deploy_dir }}/{{ container_name }} + file: + path: '{{ deploy_dir }}/{{ container_name }}' + state: absent + diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/upgrade.yml new file mode 100644 index 0000000..c5cb6f2 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/upgrade.yml @@ -0,0 +1,22 @@ +- name: Creating config tmp directory + file: + state: directory + path: "/tmp/{{ container_name }}" + +- name: Fetching config files + fetch: + src: "{{ deploy_dir }}/{{ container_name }}/{{ item}}" + dest: "/tmp/{{ container_name }}/{{ item}}" + flat: yes + loop: + - 'docker-compose.yml' + +- name: Loading variable by docker-compose.yml file + include_vars: + file: "/tmp/{{ container_name }}/docker-compose.yml" + name: docker_config + +- name: Set variable + set_fact: + saved_query_scheduler: + java_opts: "{{ docker_config['services']['{{ container_name }}']['environment']['JAVA_OPTS'] }}" diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/application.yml.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/application.yml.j2 new file mode 100644 index 0000000..607499b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/application.yml.j2 @@ -0,0 +1,20 @@ +nacos: + config: + type: yaml #配置集的配置格式 + server-addr: {{ vrrp_instance.default.virtual_ipaddress }}:8848 #配置中心地址 + namespace: {{ config_namespace }} #命名空间 + data-id: saved-query-scheduler.yml # 数据集ID + auto-refresh: true #开启自动刷新 + group: Galaxy #配置对应的分组 + username: nacos #Nacos认证用户 + password: nacos #Nacos认证密码 + bootstrap: + enable: true #开启配置预加载功能 + log: + enable: true #开启Nacos支持日志级别的加载时机 +spring: + profiles: + active: dev +logging: + config: ./config/log4j2-dev.xml + diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..7f8c0e3 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/docker-compose.yml.j2 @@ -0,0 +1,14 @@ +version: "3" +services: + saved-query-scheduler: + image: {{ image_name }}:{{ image_tag }} + container_name: {{ container_name }} + environment: + JAVA_OPTS: "{{ saved_query_scheduler.java_opts }}" + ports: + - "9093:9093" + volumes: + - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/saved-query-scheduler/config" + - "{{ deploy_dir }}/{{ container_name }}/logs:/home/tsg/galaxy/saved-query-scheduler/logs" + restart: always + network_mode: 'host' diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/log4j2-dev.xml.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/log4j2-dev.xml.j2 new file mode 100644 index 0000000..0e8de70 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/log4j2-dev.xml.j2 @@ -0,0 +1,56 @@ + + + + + + + 10M + + 10 + + error + + logs + + saved-query-scheduler + + [%d{yyyy-MM-dd HH:mm:ssZ}{UTC}] [%p] [Thread:%t] %l %x - %m%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/push_config.sh.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/push_config.sh.j2 new file mode 100644 index 0000000..a5ea43f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/push_config.sh.j2 @@ -0,0 +1,52 @@ +#!/bin/bash + +config_path_root=$1 +nacos_server={{ vrrp_instance.default.virtual_ipaddress }} + +tenant="{{ config_namespace }}" +app_name="{{ container_name }}" + +shell_result=1 + +# 推送配置 +function push() { + local config_file=$1 + local data_id="${config_file##*/}" + local suffix="${config_file##*.}" + + [ $suffix == "sql" ] && suffix="text" + [ $suffix == "yml" ] && suffix="yaml" + + local nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy" + local result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content@"$config_file") + if [[ $result -eq '200' ]]; then + sleep 2 + check=$(curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id") + #echo "$check $data_id" + + if [[ $check -ne '200' ]]; then + shell_result=0 + fi + fi +} + +# 获取本地文件 +function pushConfig() { + for config_path in $(readlink -f $config_path_root); do + if [ -d $config_path ]; then + for file in $(ls $config_path); do + config_path_file=$config_path/$file + if [ -f $config_path_file ]; then + push $config_path_file + fi + done + fi + if [ -f $config_path ]; then + push $config_path + fi + done +} + +pushConfig +echo $shell_result + diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/saved-query-scheduler.yml.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/saved-query-scheduler.yml.j2 new file mode 100644 index 0000000..7f37622 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/saved-query-scheduler.yml.j2 @@ -0,0 +1,126 @@ +#http的端口 +server: + port: 9093 +#更新进度条的时间10s +scan: + result: + scheduled: + plan: 0/15 * * * * ? +#同时间执行是线程数 +globle: + job_thread: 1 +#Hbasehttp的端口 +#Hbase的表名等配置通畅不需要更改 +hbase: + table: tsg_galaxy:saved_query_result + zookeeper_quorum: "{{ zookeeper_servers }}" + zookeeper_property_clientPort: 2181 + zookeeper_znode_parent: /hbase + client_retries_number: 3 + rpc_timeout: 100000 + connect_pool: 10 + #存入Hbase的cell级别生存时间 根据部署环境填写：1.TSG 不设置(永久有效); 2.CN 7(默认7天) 单位：Day + cell_ttl_d: + +#查询网关ip +ck: + gateway_ip: "{{ vrrp_instance.default.virtual_ipaddress }}:9999" + +#zk集群的ip +zookeeper: + connectString: "{{ zookeeper_servers }}" +#是否启用zookeeper 0启用(集群) 1禁用(单机) + open: "{{ zookeeper_open }}" + retryCount: 6 + elapsedTimeMs: 10000 + sessionTimeoutMs: 50000 + connectionTimeoutMs: 50000 + nameSpace: reportservice + +#最大连接数 +http: + maxTotal: 300 +#并发数 + defaultMaxPerRoute: 100 +#创建连接的最长时间 + connectTimeout: 10000 +#从连接池中获取到连接的最长时间 + connectionRequestTimeout: 10000 +#数据传输的最长时间 + socketTimeout: 21605000 +#提交请求前测试连接是否可用 + staleConnectionCheckEnabled: true + socketTimeoutShort: 30000 + +#mariadb的url +spring: + application: + name: saved-query-scheduler + + datasource: + url: "jdbc:mariadb://{{ vrrp_instance.default.virtual_ipaddress }}:3306/tsg_olap?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false" +#mariadb的用户名 + username: "ENC(63aTpwv2vH0vPikW+3Jjig==)" +#mariadb的密码 + password: "ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y)" + +#以下配置不需要更改通常 + name: druidDataSource + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: org.mariadb.jdbc.Driver + +#配置监控统计拦截的filters，去掉后监控界面SQL无法进行统计，’wall’用于防火墙 + druid: + filters: stat,wall,slf4j + #最大连接数 + max-active: 30 + #最小连接数 + min-idle: 1 + #初始化连接数 + initial-size: 2 + #获取连接最大超时时间 + max-wait: 600000 + #间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒 + time-between-eviction-runs-millis: 60000 + # 一个连接在池中最小生存的时间，单位是毫秒 + min-evictable-idle-time-millis: 300000 + #验证连接是否可用,在数据库中执行一条sql + validation-query: select 1 + #建议配置为true，不影响性能，并且保证安全性。申请连接的时候检测，如果空闲时间大于timeBetweenEvictionRunsMillis， + # 执行validationQuery检测连接是否有效 + test-while-idle: true + #申请连接时执行validationQuery检测连接是否有效，做了这个配置会降低性能 + test-on-borrow: true + test-on-return: false + connection-properties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 + #是否开启WebStatFilter + web-stat-filter: + enabled: true + #设置不统计哪些URL(用于排除一些不必要的url) + exclusions: '*.js,*.gif,*.jpg,*..ng,*.css,*.ico,/druid/*' + #是否开启Druid监控信息显示页面 + stat-view-servlet: + enabled: true + #甚至浏览器访问路径 + url-pattern: /druid/* + #禁止手动重置监控数据 + reset-enable: false + #durid-ui页面账户密码 + login-username: admin + login-password: admin + #Spring监控，对内部各接口调用的监控,需要导入aop相关包 + aop-patterns: com.mesa.reportservice.controller.*,com.mesa.reportservice.service.*,com.mesa.reportservice.mapper.* +mybatis: + typeAliasesPackage: com.mesa.reportservice.bean + mapperLocations: classpath*:/mappers/*.xml +management: + endpoints: + web: + exposure: + include: '*' + metrics: + tags: + application: saved-query-scheduler + +logging: + config: ./config/log4j2-dev.xml diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/vars/main.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/vars/main.yml new file mode 100644 index 0000000..a46c2d8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/vars/main.yml @@ -0,0 +1,18 @@ +software_packages_dir: "{{ role_path }}/../software-packages" +image_name: saved-query-scheduler + +image_tag: 374 + +container_name: "{{ image_name }}" + +backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/{{ container_name }}" + +backup_items: + - config + - docker-compose.yml + - init + +zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}" + +#是否启用zookeeper 0启用(集群) 1禁用(单机) +zookeeper_open: "{{ 0 if groups.saved_query_scheduler | length > 1 else 1 }}" diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/init.yml new file mode 100644 index 0000000..0bed724 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/init.yml @@ -0,0 +1,51 @@ +- name: Create directory + file: + state: directory + path: '{{ deploy_dir }}/schedule-service/{{ item }}' + loop: + - 'dos-baseline' + - 'iplearning' + - 'mmdb' + +- name: Unpack service to {{ deploy_dir }}/ + unarchive: + src: 'files/{{ item }}' + dest: '{{ deploy_dir }}/schedule-service/' + loop: + - 'dos-baseline.zip' + - 'iplearning.zip' + +- name: Copy service files + loop: + - { src: "iplearning/application.properties.j2", dest: "iplearning/application.properties", mode: "0644" } + - { src: "iplearning/start.sh.j2", dest: "iplearning/start.sh", mode: "0755" } + - { src: "dos-baseline/application.properties.j2", dest: "dos-baseline/application.properties", mode: "0644" } + - { src: "mmdb/mmdb-perl.pl.j2", dest: "mmdb/mmdb-perl.pl", mode: "0755" } + template: + src: "{{ item.src }}" + dest: "{{ deploy_dir }}/schedule-service/{{ item.dest }}" + mode: "{{ item.mode }}" + force: true + +- name: Change mode and execute iplearning upconfig.sh upconfig.sh + shell: cd "{{ deploy_dir }}/schedule-service/iplearning/" && chmod +x upconfig.sh && sh upconfig.sh + +- name: Change mode and execute dosbaseline upconfig.sh + shell: cd "{{ deploy_dir }}/schedule-service/dos-baseline/" && chmod +x upconfig.sh && sh upconfig.sh + +- name: Copying perl image + copy: + src: '{{ software_packages_dir }}/{{ perl_image_name }}-{{ perl_image_tag }}.tar' + dest: '{{ deploy_dir }}/schedule-service/mmdb' + force: true + +- name: Load perl Image + docker_image: + name: '{{ perl_image_name }}' + tag: '{{ perl_image_tag }}' + load_path: '{{ deploy_dir }}/schedule-service/mmdb/{{ perl_image_name }}-{{ perl_image_tag }}.tar' + source: load + force_tag: yes + force_source: yes + timeout: 300 + diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/main.yml new file mode 100644 index 0000000..7df2750 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/main.yml @@ -0,0 +1,9 @@ +- block: + - include: uninstall.yml + - include: init.yml + when: operation=='install' + +- block: + - include: init.yml + when: operation=='upgrade' + diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/uninstall.yml new file mode 100644 index 0000000..d2a9d36 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/uninstall.yml @@ -0,0 +1,4 @@ +- name: delete old folders + file: + state: absent + path: '{{ deploy_dir }}/schedule-service' diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/templates/dos-baseline/application.properties.j2 b/tsg-olap/parcels/roles/initialization/schedule-service/templates/dos-baseline/application.properties.j2 new file mode 100644 index 0000000..5e472e1 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/schedule-service/templates/dos-baseline/application.properties.j2 @@ -0,0 +1,85 @@ +############## 数据库配置 ############### +########################################## +#Druid配置 +druid.url=jdbc:avatica:remote:url=http://{{ vrrp_instance.default.virtual_ipaddress }}:8082/druid/v2/sql/avatica/ +druid.driver=org.apache.calcite.avatica.remote.Driver +druid.table=traffic_top_destination_ip_metrics_log + +#HBase配置 +hbase.table=dos:ddos_traffic_baselines +hbase.zookeeper.quorum={{ zookeeper_servers }} +hbase.zookeeper.client.port=2181 + +########################################## +############## Druid数据读取 ############### +########################################## +#读取druid时间范围方式， +# 0：读取默认范围天数read.historical.days； +# 1：指定时间范围 +read.druid.time.limit.type=0 +read.druid.min.time=1663430400000 +read.druid.max.time=1663603200000 + +#Druid字段映射 +druid.attacktype.tcpsynflood=TCP SYN Flood +druid.attacktype.udpflood=UDP Flood +druid.attacktype.icmpflood=ICMP Flood +druid.attacktype.dnsamplification=DNS Flood +druid.columnname.serverip=destination_ip +druid.columnname.vsysid=vsys_id +druid.columnname.attacktype=attack_type +druid.columnname.recvtime=__time +druid.columnname.partition.num=partition_num +baseline.metric.type=session_rate +#baseline存储级别，1-type1;2-type1及type2;3-all +baseline.save.level=3 + +#Hbase字段映射 +hbase.baseline.generation.type.suffix=baseline_type +hbase.baseline.zero.replace.value.suffix=default_value + +#数据情况 +#读取历史N天数据，最小值为3天（需要判断周期性） +read.historical.days=3 +#历史数据汇聚粒度为10分钟 +historical.grad=10 +# 数据库Time格式 +time.format=yyyy-MM-dd HH:mm:ss + +########################################## +############ Baseline生成参数 ############# +########################################## +baseline.range.days=1 +baseline.function=KalmanFilter +baseline.period.correlative.threshold=0.8 +baseline.historical.frequency.thread=0.5 +baseline.exception.percentile=0.99 +baseline.exception.fill.percentile=0.99 +baseline.rational.percentile=0.99 + +baseline.type2.percentile=1 +baseline.type3.percentile=1 + +# p99/p50 倍数阈值 +baseline.times.percent.times=10 + +#Kalman Filter +baseline.kalman.q=0.1 +baseline.kalman.r=0.1 +baseline.kalman.p=8 +baseline.kalman.m=2 + +monitor.frequency.bin.num=100 + +########################################## +################ 并发参数 ################# +########################################## +all.partition.num=10 +core.pool.size=10 +max.pool.size=10 +#druid分区字段partition_num的最大值为9999 +druid.statement.query.timeout=36000 +druid.partition.num.max=10000 +druid.connection.retry.time.max=10000 +#druid重连等待时间约为一个线程处理完成时间 +druid.connection.retry.sleep.time=1000 diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/application.properties.j2 b/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/application.properties.j2 new file mode 100644 index 0000000..392b6e6 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/application.properties.j2 @@ -0,0 +1,45 @@ +#spark任务配置 +spark.sql.shuffle.partitions=30 +spark.executor.memory=4g +spark.executor.cores=10 +spark.cores.max=10 +spark.local.dir=./tmp +spark.app.name=iplearning +spark.network.timeout=300s +spark.serializer=org.apache.spark.serializer.KryoSerializer +master=local[*] +#spark读取clickhouse配置 +spark.read.clickhouse.url=jdbc:clickhouse://{{ vrrp_instance.default.virtual_ipaddress }}:8124/tsg_galaxy_v3 +spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver +spark.read.clickhouse.user=default +spark.read.clickhouse.password={{ clickhouse_default_pin }} +spark.read.clickhouse.numPartitions=5 +spark.read.clickhouse.fetchsize=10000 +spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME +spark.read.clickhouse.session.table=session_record +spark.read.clickhouse.radius.table=radius_record +clickhouse.socket.timeout=300000 +#arangoDB配置 +arangoDB.host={{ groups.arangodb[0] }} +arangoDB.port=8529 +arangoDB.user=upsert +arangoDB.password={{ arangodb_upsert_pin }} +arangoDB.DB.name=tsg_galaxy_v3 +arangoDB.ttl=3600 + +thread.pool.number=10 + +#读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围 +clickhouse.time.limit.type=0 +read.clickhouse.max.time=1634902508 +read.clickhouse.min.time=1631759985 + +update.arango.batch=10000 + +distinct.client.ip.num=10000 +recent.count.hour=24 + +update.interval=3600 +arangodb.total.num=20000000 +#读取radius时间范围,与radius任务执行周期一致,单位:分钟 +read.radius.granularity=-30 diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/start.sh.j2 b/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/start.sh.j2 new file mode 100644 index 0000000..0c30868 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/start.sh.j2 @@ -0,0 +1,43 @@ +#!/bin/bash +source /etc/profile + +source /etc/profile + +jar_name="ip-learning-spark.jar" +java_opts="-Xms1g -Xmx5g" +iplearning_home=/opt/schedule-service/iplearning + +isrun=$(pgrep -f ip-learning-spark.jar | wc -l) + +#start spark job +function startJob() { + mkdir -p $iplearning_home/logs/IpLearningApplication + mkdir -p $iplearning_home/logs/IpRecommendApplication + + echo "IpLearningApplication 程序运行 - $(date "+%Y-%m-%d %H:%M:%S")" >>$iplearning_home/logs/IpLearningApplication/running-$(date +'%Y-%m').log + nohup /usr/lib/jvm/jdk1.8.0_73/bin/java -cp $iplearning_home/$jar_name $java_opts cn.ac.iie.main.IpLearningApplication >>$iplearning_home/logs/IpLearningApplication/running-$(date +'%Y-%m').log + + echo "IpRecommendApplication 程序运行 - $(date "+%Y-%m-%d %H:%M:%S")" >>$iplearning_home/logs/IpRecommendApplication/running-$(date +'%Y-%m').log + nohup /usr/lib/jvm/jdk1.8.0_73/bin/java -cp $iplearning_home/$jar_name $java_opts cn.ac.iie.main.IpRecommendApplication >>$iplearning_home/logs/IpRecommendApplication/running-$(date +'%Y-%m').log + +} + +if [[ $isrun -ge 1 ]]; then + OLD_NUM=$(cat $iplearning_home/runnum) + RESTART_NUM=$(expr $OLD_NUM + 1) + echo $RESTART_NUM >$iplearning_home/runnum + + if [ $RESTART_NUM -gt "2" ]; then + pgrep -f ip-learning-spark.jar | xargs kill -9 + startJob + + echo 1 >$iplearning_home/runnum + fi + +else + + startJob + echo 1 >$iplearning_home/runnum + nohup $iplearning_home/iplearning_monitor.sh >/dev/null 2>&1 & + +fi diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/templates/mmdb/mmdb-perl.pl.j2 b/tsg-olap/parcels/roles/initialization/schedule-service/templates/mmdb/mmdb-perl.pl.j2 new file mode 100644 index 0000000..2b2180b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/schedule-service/templates/mmdb/mmdb-perl.pl.j2 @@ -0,0 +1,146 @@ +#!/usr/bin/perl +use strict; #use this all times +use warnings; #this too - helps a lot! +use JSON; +use MaxMind::DB::Writer::Tree; +use Getopt::Long; +use utf8; +use LWP::UserAgent; +use POSIX qw(strftime); +binmode(STDIN,":encoding(gbk)"); + + +###################连接信息################### +my $serverHost="{{ cm_api }}"; +my $token="{{ cm_api_token }}"; + +#自定义：0. 内置：1 +my @args = @ARGV; +my $isBuiltin= @args?$args[0]:1; +my $url="http://$serverHost:8080/v1/global/geolocation/dict/detail?is_builtin=$isBuiltin&vsys_vim id=0"; +my $fileName=$isBuiltin==0?'ip_user_defined':'ip_builtin'; +my $outfile="$fileName.mmdb"; +my $mmdbLog="$fileName.log"; +my $mmdbStatus=$fileName.'_status.json'; +print "The perl script is running, output to $outfile \n"; + +my %types = ( + AREA_CODE => 'utf8_string', + ASN => 'utf8_string', + ISP => 'utf8_string', + LATITUDE => 'utf8_string', + LONGITUDE => 'utf8_string', + COUNTRY => 'utf8_string', + SUPER_ADMINISTRATIVE_AREA => 'utf8_string', + ADMINISTRATIVE_AREA => 'utf8_string', + SUB_ADMINISTRATIVE_AREA => 'utf8_string', + LOCALITY => 'utf8_string', + DEPENDENT_LOCALITY => 'utf8_string', + DOUBLE_DEPENDENT_LOCALITY => 'utf8_string', +); + +# V6databse info +my $treeV6 = MaxMind::DB::Writer::Tree->new( + ip_version => 6, + record_size => 24, + database_type => 'IPLOCATOR', + languages => ['en'], + description => { en => 'My database of IP data' }, + map_key_type_callback => sub { $types{ $_[0] } }, + remove_reserved_networks => 0, +); + +my $pageNo = 1; +my $pageSize = 2000; +my $total= 0; +my $requestMsg= ''; +my $requestCode= 0; +my @ipSample= (); + +my $ua = new LWP::UserAgent; +while(1==1){ +my $request = new HTTP::Request("GET", "$url&page_size=$pageSize&page_no=$pageNo"); +$request->header('Authorization' => $token); +my $response = $ua->request($request); +my $json = new JSON; +my $obj = $json->decode($response->content); + +$requestCode=$obj->{"code"}; +$requestMsg=$obj->{"msg"}; +my $dbTotal=$obj->{"data"}->{"total"}; + +#记录日志 +open(MMDBLOG, "+>>$mmdbLog"); +my $gmtDatestring = strftime "%Y-%m-%d %H:%M:%S", gmtime; + +my $list=$obj->{"data"}->{"list"}; +my $JSON =JSON->new->allow_nonref; +my $data = $JSON->encode($list); +my $arraydata=decode_json($data); +foreach my $ref (@$arraydata) { + $treeV6->insert_range( + $ref->{'start_ip'}, + $ref->{'end_ip'}, + { + AREA_CODE => isNotBlank($ref->{'areacode'}), + ASN => isNotBlank($ref->{'asnumber'}), + ISP => isNotBlank($ref->{'isp'}), + LATITUDE => isNotBlank($ref->{'latitude'}), + LONGITUDE => isNotBlank($ref->{'longitude'}), + COUNTRY => changeSplit(isNotBlank($ref->{'country_region'})), + SUPER_ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'super_administrative_area'})), + ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'administrative_area'})), + SUB_ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'sub_administrative_area'})), + LOCALITY => changeSplit(isNotBlank($ref->{'locality'})), + DEPENDENT_LOCALITY => changeSplit(isNotBlank($ref->{'dependent_locality'})), + DOUBLE_DEPENDENT_LOCALITY => changeSplit(isNotBlank($ref->{'double_dependent_locality'})), + }, + ); + $total+=1; +} +if(@$arraydata){ + push(@ipSample, @$arraydata[0]->{'start_ip'}); +} +my $percentage = $dbTotal==0?0 : int(($total/$dbTotal) * 100); +print "$gmtDatestring MMDB request $requestMsg pageNum: $pageNo, queried/total: $total/$dbTotal, progress: $percentage% .\n"; +print MMDBLOG "$gmtDatestring MMDB request $requestMsg pageNum: $pageNo, queried/total: $total/$dbTotal, progress: $percentage% .\n"; +$pageNo+=1; +if(@$arraydata < $pageSize){ + last; + } +} + +if($total>0){ + open my $fhV6, '>:raw', $outfile; + $treeV6->write_tree($fhV6); + close($outfile); +} + +my %statusHash = ('status'=>$requestCode, 'updateTime'=>time(), 'total'=>$total, 'message'=>$requestMsg,'ipSample'=> \@ipSample); +my $statusJson = encode_json \%statusHash; +open my $statusFile, '>', $mmdbStatus; +print $statusFile $statusJson; + +my $gmtDateLog = strftime "%Y-%m-%d %H:%M:%S", gmtimvim m e; +print MMDBLOG "$gmtDateLog MMDB IP Write Over. \n\n"; +close(MMDBLOG); +print "The perl script is finished!\n"; + +sub changeSplit { + my $n = scalar(@_); + my $i = $_[0]; + $i =~ s/\\b/ /g; + + return "$i" +} + +sub isNotBlank { + my $tmp = $_[0]; + if($tmp){ + return $tmp; + } else{ + return " "; + } +} + + diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/schedule-service/vars/main.yml new file mode 100644 index 0000000..5cc1bd9 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/schedule-service/vars/main.yml @@ -0,0 +1,7 @@ +software_packages_dir: "{{ role_path }}/../software-packages" + +zk_servers_noport: "{% for host in groups['zookeeper'] %}{{ host }}{% if not loop.last %},{% endif %}{% endfor %}" +zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}" + +perl_image_name: perl +perl_image_tag: v3.1 diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/backup.yml new file mode 100644 index 0000000..1103fa8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/backup.yml @@ -0,0 +1,25 @@ +- name: Check if backup directory exists + stat: + path: "{{ backup_path }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/flink/{{ item }}" + dest: "{{ backup_path }}" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}" + state: absent + when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/bak.check.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/bak.check.yml new file mode 100644 index 0000000..65ff569 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/bak.check.yml @@ -0,0 +1,27 @@ +- block: + - name: Checking the number of Flink tasks + shell: "source /etc/profile && sleep 30 && yarn application --list | grep -v 'Total number of applications' | grep RUNNING | wc -l" + register: jobNums + + - name: Checking the number of Flink tasks + fail: + msg: "The number of Flink running tasks does not match the record.The browser enter {{ groups.yarn[0] }}:8080 for verification." + when: jobNums.stdout != '11' + run_once: true + delegate_facts: true + delegate_to: "{{ groups.yarn[0] }}" + when: (groups.yarn|length ) > 1 + +- block: + - name: Checking the number of Flink tasks + shell: "source /etc/profile && sleep 30 && flink list | grep '(RUNNING)' | wc -l" + register: jobNums + + - name: Checking the number of Flink tasks + fail: + msg: "The number of Flink running tasks does not match the record.The browser enter {{ groups.yarn[0] }}:8080 for verification." + when: jobNums.stdout != '11' + run_once: true + delegate_facts: true + delegate_to: "{{ groups.yarn[0] }}" + when: (groups.yarn|length ) == 1 diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/check.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/check.yml new file mode 100644 index 0000000..020e1dd --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/check.yml @@ -0,0 +1,65 @@ +- name: Get init task template + find: + paths: "templates" + patterns: "config" + file_type: directory + recurse: yes + register: find_dirctory + run_once: true + delegate_facts: true + delegate_to: 127.0.0.1 + +- set_fact: + find_config: "{{ find_dirctory.files | map(attribute='path') | list }}" + +- name: Find config template + find: + paths: "{{ item }}" + file_type: "file" + recurse: yes + with_list: "{{ find_config }}" + register: find_file + run_once: true + delegate_facts: true + delegate_to: 127.0.0.1 + +- set_fact: + init_task_list: "{{ find_file.results | json_query('[].files[].path') | map('basename') | map('regex_replace', '^(.*)\\.j2$', '\\1') | list }}" + +- block: + - name: Gets the running yarn task + shell: "source /etc/profile && sleep 60 && yarn application --list -appStates RUNNING | grep -v 'Total number of applications' | grep RUNNING| awk '{print $2}'" + register: run_task_list + + - debug: var=run_task_list.stdout_lines + + - name: Checking the number of Flink task + fail: + msg: "The number of Flink running tasks does not match the record.The browser enter {{ groups.yarn[0] }}:8080 for verification." + when: run_task_list.stdout_lines | intersect(init_task_list) |length != (init_task_list|length) + run_once: true + delegate_facts: true + delegate_to: "{{ groups.yarn[0] }}" + when: task_mode == 'yarn-per-job' + +- block: + - name: Gets the running yarn task + shell: source /etc/profile && sleep 60 && yarn application --list -appStates RUNNING | grep -v 'Total number of applications' | grep RUNNING| awk '{print $1}' + register: run_application_id + + - debug: var=run_application_id.stdout + + - name: Gets the running yarn task + shell: curl -s http://{{ groups.yarn[0] }}:8080/proxy/{{ run_application_id.stdout }}/jobs/overview| jq -r '.jobs[] | select(.state=="RUNNING") | .name' + register: run_task_list + + - debug: var=run_task_list.stdout_lines + + - name: Checking the number of Flink task + fail: + msg: "The number of Flink running tasks does not match the record.The browser enter {{ groups.yarn[0] }}:8080 for verification." + when: run_task_list.stdout_lines | intersect(init_task_list) |length != (init_task_list|length) + run_once: true + delegate_facts: true + delegate_to: "{{ groups.yarn[0] }}" + when: task_mode == 'yarn-session' diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/init.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/init.yml new file mode 100644 index 0000000..d400056 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/init.yml @@ -0,0 +1,154 @@ +- name: Set tasks directory variable + set_fact: + task_dir: + - app-protocol-stat-traffic-merge + - dos-detection + - file-chunk-combiner + - groot-stream + - sip-rtp-correlation + - topn-metrics-job + sub_dir: + - bin + - config + - env + - lib + +- name: Create yarn session + shell: source /etc/profile && yarn-session.sh -d + run_once: true + when: task_mode == 'yarn-session' + +- name: Create {{ deploy_dir }}/flink/topology + file: + path: "{{ deploy_dir }}/flink/topology" + state: directory + +- name: Unpack Knowledge base files + unarchive: + src: 'files/data.zip' + dest: '{{ deploy_dir }}/flink/topology/' + +- block: + - name: Create {{ deploy_dir }}/flink/topology + file: + path: "{{ deploy_dir }}/flink/topology/{{ item.0 }}/{{ item.1 }}" + state: directory + with_nested: + - "{{ task_dir }}" + - "{{ sub_dir }}" + changed_when: false + + - name: Unpack Knowledge base files + unarchive: + src: 'files/data.zip' + dest: '{{ deploy_dir }}/flink/topology/' + + - name: Copying upload_knowlodge.sh + template: + src: 'upload_knowlodge.sh.j2' + dest: '{{ deploy_dir }}/flink/topology/upload_knowlodge.sh' + force: true + mode: 0755 + + - name: upload Knowledge + shell: cd {{ deploy_dir }}/flink/topology/ && sh upload_knowlodge.sh + run_once: true + + - name: Copying topology to {{ deploy_dir }}/ + copy: + src: 'files/topology' + dest: '{{ deploy_dir }}/flink/' + force: true + + - name: Copying jobs config files + template: + src: "{{ item.name }}/config/{{ item.config }}.j2" + dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/config/{{ item.config }}" + force: true + with_items: + - { name: 'app-protocol-stat-traffic-merge', config: 'agg_app_protocol_traffic' } + - { name: 'dos-detection', config: 'detection_dos_attack' } + - { name: 'file-chunk-combiner', config: 'agg_traffic_file_chunk_combine' } + - { name: 'file-chunk-combiner', config: 'agg_troubleshooting_file_chunk_combine' } + - { name: 'sip-rtp-correlation', config: 'correlation_sip_rtp_session' } + - { name: 'topn-metrics-job', config: 'agg_session_record_topn' } + + - name: Copying jobs env files + template: + src: "{{ item.name }}/env/{{ item.config }}.sh" + dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/env/{{ item.config }}.sh" + mode: 0755 + force: true + with_items: + - { name: 'app-protocol-stat-traffic-merge', config: 'agg_app_protocol_traffic' } + - { name: 'dos-detection', config: 'detection_dos_attack' } + - { name: 'file-chunk-combiner', config: 'agg_traffic_file_chunk_combine' } + - { name: 'file-chunk-combiner', config: 'agg_troubleshooting_file_chunk_combine' } + - { name: 'sip-rtp-correlation', config: 'correlation_sip_rtp_session' } + - { name: 'topn-metrics-job', config: 'agg_session_record_topn' } + + - name: Copying jobs start.sh + template: + src: "start.sh" + dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/bin/start.sh" + force: true + mode: 0755 + with_items: + - { name: 'app-protocol-stat-traffic-merge' } + - { name: 'dos-detection' } + - { name: 'file-chunk-combiner' } + - { name: 'sip-rtp-correlation' } + - { name: 'topn-metrics-job' } + + - name: Copying jobs stop.sh + template: + src: "stop.sh" + dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/bin/stop.sh" + force: true + mode: 0755 + with_items: + - { name: 'app-protocol-stat-traffic-merge' } + - { name: 'dos-detection' } + - { name: 'file-chunk-combiner' } + - { name: 'groot-stream' } + - { name: 'sip-rtp-correlation' } + - { name: 'topn-metrics-job' } + + - name: Copying groot start.sh + template: + src: "{{ item.name }}/start.sh" + dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/bin/start.sh" + force: true + mode: 0755 + with_items: + - { name: 'groot-stream' } + + - name: Copying groot config templates + template: + src: "{{ item }}" + dest: "{{ deploy_dir }}/flink/topology/groot-stream/config/{{ item | basename | regex_replace('.j2$', '') }}" + mode: "0644" + force: true + with_fileglob: "templates/groot-stream/config/*" + + - name: Copying groot env templates + template: + src: "{{ item }}" + dest: "{{ deploy_dir }}/flink/topology/groot-stream/env/{{ item | basename }}" + mode: "0755" + force: true + with_fileglob: "templates/groot-stream/env/*" + + - name: Starting flink tasks + shell: cd '{{ deploy_dir }}/flink/topology/{{ item.name }}/bin' && chmod +x *.sh && ./stop.sh && ./start.sh + with_items: + - { name: 'groot-stream' } + - { name: 'app-protocol-stat-traffic-merge' } + - { name: 'dos-detection' } + - { name: 'file-chunk-combiner' } + - { name: 'sip-rtp-correlation' } + - { name: 'topn-metrics-job' } + run_once: true + + when: inventory_hostname in groups['yarn'][:2] + diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/main.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/main.yml new file mode 100644 index 0000000..769fd2b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/main.yml @@ -0,0 +1,16 @@ +- block: + - include: uninstall.yml + - include: init.yml + - include: check.yml + when: (operation) == "install" + +- block: + - include: backup.yml + when: inventory_hostname in groups['yarn'][:2] + - include: uninstall.yml + - include: init.yml + when: (operation) == "upgrade" + +- block: + - include: uninstall.yml + when: (operation) == "uninstall" and inventory_hostname in groups['yarn'][:2] diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/uninstall.yml new file mode 100644 index 0000000..4860ee0 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/uninstall.yml @@ -0,0 +1,14 @@ +- name: Check {{ deploy_dir }}/flink/topology + stat: + path: "{{ deploy_dir }}/flink/topology" + register: directory_stat + +- name: Stop all jobs + raw: yarn application -list | awk '/application_/ {print $1}' | xargs -I {} yarn application -kill {} + run_once: true + +- name: Delete {{ deploy_dir }}/flink/topology + file: + path: "{{ deploy_dir }}/flink/topology" + state: absent + when: directory_stat.stat.exists diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/config/agg_app_protocol_traffic.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/config/agg_app_protocol_traffic.j2 new file mode 100644 index 0000000..a102c7b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/config/agg_app_protocol_traffic.j2 @@ -0,0 +1,27 @@ +#kafka 接收数据topic +source.kafka.topic=NETWORK-TRAFFIC-METRIC + +source.kafka.props.bootstrap.servers={{ kafka_source_servers }} + +source.kafka.props.group.id=app-protocol-merge-230510-1 + +source.kafka.props.security.protocol=SASL_PLAINTEXT + +source.kafka.props.sasl.mechanism=PLAIN + +source.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + +#补全数据 输出 topic +sink.kafka.topic=NETWORK-TRAFFIC-METRIC + +sink.kafka.props.bootstrap.servers={{ kafka_sink_servers }} + +sink.kafka.props.security.protocol=SASL_PLAINTEXT + +sink.kafka.props.sasl.mechanism=PLAIN + +sink.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + +count.window.time=5 + +watermark.max.orderness=5 diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/env/agg_app_protocol_traffic.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/env/agg_app_protocol_traffic.sh new file mode 100644 index 0000000..b3e1b15 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/env/agg_app_protocol_traffic.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.topology.ApplicationProtocolTopology" +export PARALLELISM="{{ flink.agg_app_protocol_traffic['parallelism'] }}" +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.agg_app_protocol_traffic | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/config/detection_dos_attack.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/config/detection_dos_attack.j2 new file mode 100644 index 0000000..85bc166 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/config/detection_dos_attack.j2 @@ -0,0 +1,50 @@ +#kafka source +source.kafka.topic=DOS-SKETCH-RECORD +source.kafka.props.bootstrap.servers={{ kafka_source_servers }} +source.kafka.props.group.id=dos-detection-job-20240116 +source.kafka.props.session.timeout.ms=60000 +source.kafka.props.max.poll.records=5000 +source.kafka.props.max.partition.fetch.bytes=31457280 +source.kafka.props.security.protocol=SASL_PLAINTEXT +source.kafka.props.sasl.mechanism=PLAIN +source.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin"password="galaxy2019"; + +#kafka sink +kafka.sink.event.topic.name=DOS-EVENT +kafka.sink.metric.topic=TRAFFIC-TOP-DESTINATION-IP-METRIC +sink.kafka.props.bootstrap.servers={{ kafka_sink_servers }} +sink.kafka.props.security.protocol=SASL_PLAINTEXT +sink.kafka.props.sasl.mechanism=PLAIN +sink.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; +sink.kafka.props.acks=1 +sink.kafka.props.retries=0 +sink.kafka.props.linger.ms=10 +sink.kafka.props.request.timeout.ms=30000 +sink.kafka.props.batch.size=262144 +sink.kafka.props.buffer.memory=134217728 +sink.kafka.props.max.request.size=10485760 +sink.kafka.props.compression.type=snappy + +#zookeeper地址 +hbase.zookeeper.quorum={{ zookeeper_servers }} + +flink.watermark.max.orderness=300 + +#计算窗口大小，默认600s +flink.window.max.time=600 + +#cm服务访问地址 +bifang.server.uri=http://{{ cm_api }} + +knowledge.base.uri=http://{{ vrrp_instance.default.virtual_ipaddress }}:9999 +############################## 阈值 配置 ###################################### +static.sensitivity.threshold=1 +#基线敏感阈值 +baseline.sensitivity.threshold=0.2 +#基于baseline判定dos攻击的上下限 +baseline.sessions.minor.threshold=0.2 +baseline.sessions.warning.threshold=1 +baseline.sessions.major.threshold=2.5 +baseline.sessions.severe.threshold=5 +baseline.sessions.critical.threshold=8 + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/env/detection_dos_attack.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/env/detection_dos_attack.sh new file mode 100644 index 0000000..1e478c9 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/env/detection_dos_attack.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.main.DosDetectionApplication" +export PARALLELISM="{{ flink.detection_dos_attack['parallelism'] }}" +export TASK_MODE="{{ task_mode }}" + + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.detection_dos_attack | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_traffic_file_chunk_combine.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_traffic_file_chunk_combine.j2 new file mode 100644 index 0000000..c2fb093 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_traffic_file_chunk_combine.j2 @@ -0,0 +1,41 @@ +flink.job.name=agg_traffic_file_chunk_combine + +#source相关配置 +source.kafka.parallelism=1 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker={{ kafka_source_servers }} +source.kafka.group.id=agg_traffic_file_chunk_combine +source.kafka.topic=TRAFFIC-FILE-STREAM-RECORD +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=1000 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=admin +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2019 +#SSL需要 +source.kafka.tools.library={{ deploy_dir }}/topology/data/ + +parse.message.pack.parallelism=1 +#窗口相关配置 +combiner.window.parallelism=1 +combiner.window.time=10 +combiner.window.idle.time=5 +combiner.window.key.max.chunk=100000 + +#hos sink相关配置 +sink.hos.parallelism=1 +sink.hos.endpoint=http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos +sink.hos.bucket=traffic_file_bucket +sink.hos.token={{ hos_token }} +sink.hos.http.error.retry=3 +sink.hos.http.max.total=2000 +sink.hos.http.max.per.route=1000 +sink.hos.http.connect.timeout=10000 +sink.hos.http.request.timeout=10000 +sink.hos.http.socket.timeout=60000 diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_troubleshooting_file_chunk_combine.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_troubleshooting_file_chunk_combine.j2 new file mode 100644 index 0000000..af3c767 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_troubleshooting_file_chunk_combine.j2 @@ -0,0 +1,41 @@ +flink.job.name=agg_troubleshooting_file_chunk_combine + +#source相关配置 +source.kafka.parallelism=1 +#9092为无验证 9095为ssl 9094为sasl +source.kafka.broker={{ kafka_source_servers }} +source.kafka.group.id=agg_troubleshooting_file_chunk_combine +source.kafka.topic=TROUBLESHOOTING-FILE-STREAM-RECORD +#earliest从头开始 latest最新 +source.kafka.auto.offset.reset=latest +source.kafka.session.timeout.ms=60000 +#每次拉取操作从分区中获取的最大记录数 +source.kafka.max.poll.records=100 +#消费者从单个分区中一次性获取的最大字节数 +source.kafka.max.partition.fetch.bytes=31457280 +source.kafka.enable.auto.commit=true +#kafka SASL验证用户名 +source.kafka.user=admin +#kafka SASL及SSL验证密码 +source.kafka.pin=galaxy2019 +#SSL需要 +source.kafka.tools.library={{ deploy_dir }}/topology/data/ + +parse.message.pack.parallelism=1 +#窗口相关配置 +combiner.window.parallelism=1 +combiner.window.time=5 +combiner.window.idle.time=5 +combiner.window.key.max.chunk=100000 + +#hos sink相关配置 +sink.hos.parallelism=1 +sink.hos.endpoint=http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos +sink.hos.bucket=troubleshooting_file_bucket +sink.hos.token={{ hos_token }} +sink.hos.http.error.retry=3 +sink.hos.http.max.total=2000 +sink.hos.http.max.per.route=1000 +sink.hos.http.connect.timeout=10000 +sink.hos.http.request.timeout=10000 +sink.hos.http.socket.timeout=60000 diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_traffic_file_chunk_combine.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_traffic_file_chunk_combine.sh new file mode 100644 index 0000000..551535f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_traffic_file_chunk_combine.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" +export PARALLELISM="{{ flink.agg_traffic_file_chunk_combine['parallelism'] }}" +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.agg_traffic_file_chunk_combine | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_troubleshooting_file_chunk_combine.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_troubleshooting_file_chunk_combine.sh new file mode 100644 index 0000000..e01f666 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_troubleshooting_file_chunk_combine.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.FileChunkCombiner" + +export PARALLELISM="{{ flink.agg_troubleshooting_file_chunk_combine['parallelism'] }}" +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.agg_troubleshooting_file_chunk_combine | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/dos_event_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/dos_event_kafka_to_clickhouse new file mode 100644 index 0000000..1ab988c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/dos_event_kafka_to_clickhouse @@ -0,0 +1,53 @@ +sources: + kafka_source: + type : kafka + # fields: # [array of object] Field List, if not set, all fields(Map) will be output. + # watermark_timestamp: common_recv_time # [string] Watermark Field Name + # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms + # watermark_lag: 60 # [number] Watermark Lag, default is 60 + properties: + topic: DOS-EVENT + kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.session.timeout.ms: 60000 + kafka.max.poll.records: 3000 + kafka.max.partition.fetch.bytes: 31457280 + kafka.security.protocol: SASL_PLAINTEXT + kafka.ssl.keystore.location: + kafka.ssl.keystore.password: + kafka.ssl.truststore.location: + kafka.ssl.truststore.password: + kafka.ssl.key.password: + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.buffer.memory: + kafka.group.id: dos_event_kafka_to_clickhouse-20231221 + kafka.auto.offset.reset: latest + kafka.max.request.size: + kafka.compression.type: none + format: json + + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: {{ clickhouse_servers }} + table: tsg_galaxy_v3.dos_event_local + batch.size: 100000 + batch.interval: 30s + connection.user: default + connection.password: galaxy2019 + + +application: + + env: # [object] Environment Variables + name: dos_event_kafka_to_clickhouse # [string] Job Name + pipeline: + object-reuse: true # [boolean] Object Reuse, default is false + topology: + - name: kafka_source + downstream: [clickhouse_sink] + - name: clickhouse_sink + + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_proxy_event_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_proxy_event_kafka_to_clickhouse new file mode 100644 index 0000000..a09b036 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_proxy_event_kafka_to_clickhouse @@ -0,0 +1,161 @@ +sources: + kafka_source: + type : kafka + # fields: # [array of object] Field List, if not set, all fields(Map) will be output. + # watermark_timestamp: common_recv_time # [string] Watermark Field Name + # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms + # watermark_lag: 60 # [number] Watermark Lag, default is 60 + properties: + topic: PROXY-EVENT + kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.session.timeout.ms: 60000 + kafka.max.poll.records: 3000 + kafka.max.partition.fetch.bytes: 31457280 + kafka.security.protocol: SASL_PLAINTEXT + kafka.ssl.keystore.location: + kafka.ssl.keystore.password: + kafka.ssl.truststore.location: + kafka.ssl.truststore.password: + kafka.ssl.key.password: + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.buffer.memory: + kafka.group.id: etl_proxy_event_kafka_to_clickhouse-20231221 + kafka.auto.offset.reset: latest + kafka.max.request.size: + kafka.compression.type: none + format: json + +processing_pipelines: + etl_processor: # [object] Processing Pipeline + type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + remove_fields: + output_fields: + properties: + key: value + functions: # [array of object] Function List + + - function: ASN_LOOKUP + lookup_fields: [ server_ip ] + output_fields: [ server_asn ] + parameters: + option: IP_TO_ASN + kb_name: tsg_ip_asn + + - function: ASN_LOOKUP + lookup_fields: [ client_ip ] + output_fields: [ client_asn ] + parameters: + option: IP_TO_ASN + kb_name: tsg_ip_asn + + - function: SNOWFLAKE_ID + lookup_fields: [ '' ] + output_fields: [ log_id ] + parameters: + data_center_id_num: 1 + + - function: JSON_EXTRACT + lookup_fields: [ device_tag ] + output_fields: [ data_center ] + filter: + parameters: + value_expression: $.tags[?(@.tag=='data_center')][0].value + + - function: JSON_EXTRACT + lookup_fields: [ device_tag ] + output_fields: [ device_group ] + filter: + parameters: + value_expression: $.tags[?(@.tag=='device_group')][0].value + + - function: CURRENT_UNIX_TIMESTAMP + output_fields: [ processing_time ] + parameters: + precision: seconds + + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [ __timestamp ] + output_fields: [ recv_time ] + parameters: + precision: seconds + + - function: EVAL + output_fields: [ ingestion_time ] + parameters: + value_expression: 'recv_time' + + - function: DOMAIN + lookup_fields: [ http_host,ssl_sni,dtls_sni,quic_sni ] + output_fields: [ server_domain ] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + + - function: BASE64_DECODE_TO_STRING + output_fields: [mail_subject] + parameters: + value_field: mail_subject + charset_field: mail_subject_charset + + - function: BASE64_DECODE_TO_STRING + output_fields: [mail_attachment_name] + parameters: + value_field: mail_attachment_name + charset_field: mail_attachment_name_charset + + - function: PATH_COMBINE + lookup_fields: [ rtp_pcap_path ] + output_fields: [ rtp_pcap_path ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path ] + + - function: PATH_COMBINE + lookup_fields: [ http_request_body ] + output_fields: [ http_request_body ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_request_body ] + + - function: PATH_COMBINE + lookup_fields: [ http_response_body ] + output_fields: [ http_response_body ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_response_body ] + + - function: PATH_COMBINE + lookup_fields: [ mail_eml_file ] + output_fields: [ mail_eml_file ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file ] + + - function: PATH_COMBINE + lookup_fields: [ packet_capture_file ] + output_fields: [ packet_capture_file ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file ] + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: {{ clickhouse_servers }} + table: tsg_galaxy_v3.proxy_event_local + batch.size: 100000 + batch.interval: 30s + connection.user: default + connection.password: galaxy2019 + + +application: + + env: # [object] Environment Variables + name: etl_proxy_event_kafka_to_clickhouse # [string] Job Name + pipeline: + object-reuse: true # [boolean] Object Reuse, default is false + topology: + - name: kafka_source + downstream: [etl_processor] + - name: etl_processor + downstream: [clickhouse_sink] + - name: clickhouse_sink + + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_session_record_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_session_record_kafka_to_clickhouse new file mode 100644 index 0000000..9812e0b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_session_record_kafka_to_clickhouse @@ -0,0 +1,161 @@ +sources: + kafka_source: + type : kafka + # fields: # [array of object] Field List, if not set, all fields(Map) will be output. + # watermark_timestamp: common_recv_time # [string] Watermark Field Name + # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms + # watermark_lag: 60 # [number] Watermark Lag, default is 60 + properties: + topic: SESSION-RECORD + kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.session.timeout.ms: 60000 + kafka.max.poll.records: 3000 + kafka.max.partition.fetch.bytes: 31457280 + kafka.security.protocol: SASL_PLAINTEXT + kafka.ssl.keystore.location: + kafka.ssl.keystore.password: + kafka.ssl.truststore.location: + kafka.ssl.truststore.password: + kafka.ssl.key.password: + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.buffer.memory: + kafka.group.id: etl_session_record_kafka_to_clickhouse-20231229 + kafka.auto.offset.reset: latest + kafka.max.request.size: + kafka.compression.type: none + format: json + +processing_pipelines: + etl_processor: # [object] Processing Pipeline + type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + remove_fields: + output_fields: + properties: + key: value + functions: # [array of object] Function List + + - function: ASN_LOOKUP + lookup_fields: [ server_ip ] + output_fields: [ server_asn ] + parameters: + option: IP_TO_ASN + kb_name: tsg_ip_asn + + - function: ASN_LOOKUP + lookup_fields: [ client_ip ] + output_fields: [ client_asn ] + parameters: + option: IP_TO_ASN + kb_name: tsg_ip_asn + + - function: SNOWFLAKE_ID + lookup_fields: [ '' ] + output_fields: [ log_id ] + parameters: + data_center_id_num: 1 + + - function: JSON_EXTRACT + lookup_fields: [ device_tag ] + output_fields: [ data_center ] + filter: + parameters: + value_expression: $.tags[?(@.tag=='data_center')][0].value + + - function: JSON_EXTRACT + lookup_fields: [ device_tag ] + output_fields: [ device_group ] + filter: + parameters: + value_expression: $.tags[?(@.tag=='device_group')][0].value + + - function: CURRENT_UNIX_TIMESTAMP + output_fields: [ processing_time ] + parameters: + precision: seconds + + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [ __timestamp ] + output_fields: [ recv_time ] + parameters: + precision: seconds + + - function: EVAL + output_fields: [ ingestion_time ] + parameters: + value_expression: 'recv_time' + + - function: DOMAIN + lookup_fields: [ http_host,ssl_sni,dtls_sni,quic_sni ] + output_fields: [ server_domain ] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + + - function: BASE64_DECODE_TO_STRING + output_fields: [mail_subject] + parameters: + value_field: mail_subject + charset_field: mail_subject_charset + + - function: BASE64_DECODE_TO_STRING + output_fields: [mail_attachment_name] + parameters: + value_field: mail_attachment_name + charset_field: mail_attachment_name_charset + + - function: PATH_COMBINE + lookup_fields: [ rtp_pcap_path ] + output_fields: [ rtp_pcap_path ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path ] + + - function: PATH_COMBINE + lookup_fields: [ http_request_body ] + output_fields: [ http_request_body ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_request_body ] + + - function: PATH_COMBINE + lookup_fields: [ http_response_body ] + output_fields: [ http_response_body ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_response_body ] + + - function: PATH_COMBINE + lookup_fields: [ mail_eml_file ] + output_fields: [ mail_eml_file ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file ] + + - function: PATH_COMBINE + lookup_fields: [ packet_capture_file ] + output_fields: [ packet_capture_file ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file ] + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: {{ clickhouse_servers }} + table: tsg_galaxy_v3.session_record_local + batch.size: 100000 + batch.interval: 30s + connection.user: default + connection.password: galaxy2019 + + +application: + + env: # [object] Environment Variables + name: etl_session_record_kafka_to_clickhouse # [string] Job Name + pipeline: + object-reuse: true # [boolean] Object Reuse, default is false + topology: + - name: kafka_source + downstream: [etl_processor] + - name: etl_processor + downstream: [clickhouse_sink] + - name: clickhouse_sink + + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_transaction_record_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_transaction_record_kafka_to_clickhouse new file mode 100644 index 0000000..21af021 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_transaction_record_kafka_to_clickhouse @@ -0,0 +1,159 @@ +sources: + kafka_source: + type : kafka + # fields: # [array of object] Field List, if not set, all fields(Map) will be output. + # watermark_timestamp: common_recv_time # [string] Watermark Field Name + # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms + # watermark_lag: 60 # [number] Watermark Lag, default is 60 + properties: + topic: TRANSACTION-RECORD + kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.session.timeout.ms: 60000 + kafka.max.poll.records: 3000 + kafka.max.partition.fetch.bytes: 31457280 + kafka.security.protocol: SASL_PLAINTEXT + kafka.ssl.keystore.location: + kafka.ssl.keystore.password: + kafka.ssl.truststore.location: + kafka.ssl.truststore.password: + kafka.ssl.key.password: + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.buffer.memory: + kafka.group.id: etl_transaction_record_kafka_to_clickhouse-20231221 + kafka.auto.offset.reset: latest + kafka.max.request.size: + kafka.compression.type: none + format: json + +processing_pipelines: + etl_processor: # [object] Processing Pipeline + type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + remove_fields: + output_fields: + properties: + key: value + functions: # [array of object] Function List + + - function: ASN_LOOKUP + lookup_fields: [ server_ip ] + output_fields: [ server_asn ] + parameters: + option: IP_TO_ASN + kb_name: tsg_ip_asn + + - function: ASN_LOOKUP + lookup_fields: [ client_ip ] + output_fields: [ client_asn ] + parameters: + option: IP_TO_ASN + kb_name: tsg_ip_asn + + - function: SNOWFLAKE_ID + lookup_fields: [ '' ] + output_fields: [ log_id ] + parameters: + data_center_id_num: 1 + + - function: JSON_EXTRACT + lookup_fields: [ device_tag ] + output_fields: [ data_center ] + filter: + parameters: + value_expression: $.tags[?(@.tag=='data_center')][0].value + + - function: JSON_EXTRACT + lookup_fields: [ device_tag ] + output_fields: [ device_group ] + filter: + parameters: + value_expression: $.tags[?(@.tag=='device_group')][0].value + + - function: CURRENT_UNIX_TIMESTAMP + output_fields: [ processing_time ] + parameters: + precision: seconds + + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [ __timestamp ] + output_fields: [ recv_time ] + parameters: + precision: seconds + + - function: EVAL + output_fields: [ ingestion_time ] + parameters: + value_expression: 'recv_time' + + - function: DOMAIN + lookup_fields: [ http_host,ssl_sni,dtls_sni,quic_sni ] + output_fields: [ server_domain ] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + + - function: BASE64_DECODE_TO_STRING + output_fields: [mail_subject] + parameters: + value_field: mail_subject + charset_field: mail_subject_charset + + - function: BASE64_DECODE_TO_STRING + output_fields: [mail_attachment_name] + parameters: + value_field: mail_attachment_name + charset_field: mail_attachment_name_charset + + - function: PATH_COMBINE + lookup_fields: [ rtp_pcap_path ] + output_fields: [ rtp_pcap_path ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path ] + + - function: PATH_COMBINE + lookup_fields: [ http_request_body ] + output_fields: [ http_request_body ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_request_body ] + + - function: PATH_COMBINE + lookup_fields: [ http_response_body ] + output_fields: [ http_response_body ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_response_body ] + + - function: PATH_COMBINE + lookup_fields: [ mail_eml_file ] + output_fields: [ mail_eml_file ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file ] + + - function: PATH_COMBINE + lookup_fields: [ packet_capture_file ] + output_fields: [ packet_capture_file ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file ] + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: {{ clickhouse_servers }} + table: tsg_galaxy_v3.transaction_record_local + batch.size: 100000 + batch.interval: 30s + connection.user: default + connection.password: galaxy2019 + + +application: + + env: # [object] Environment Variables + name: etl_transaction_record_kafka_to_clickhouse # [string] Job Name + pipeline: + object-reuse: true # [boolean] Object Reuse, default is false + topology: + - name: kafka_source + downstream: [etl_processor] + - name: etl_processor + downstream: [clickhouse_sink] + - name: clickhouse_sink diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_voip_record_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_voip_record_kafka_to_clickhouse new file mode 100644 index 0000000..effb6db --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_voip_record_kafka_to_clickhouse @@ -0,0 +1,161 @@ +sources: + kafka_source: + type : kafka + # fields: # [array of object] Field List, if not set, all fields(Map) will be output. + # watermark_timestamp: common_recv_time # [string] Watermark Field Name + # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms + # watermark_lag: 60 # [number] Watermark Lag, default is 60 + properties: + topic: VOIP-CONVERSATION-RECORD + kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.session.timeout.ms: 60000 + kafka.max.poll.records: 3000 + kafka.max.partition.fetch.bytes: 31457280 + kafka.security.protocol: SASL_PLAINTEXT + kafka.ssl.keystore.location: + kafka.ssl.keystore.password: + kafka.ssl.truststore.location: + kafka.ssl.truststore.password: + kafka.ssl.key.password: + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.buffer.memory: + kafka.group.id: etl_voip_record_kafka_to_clickhouse-20231221 + kafka.auto.offset.reset: latest + kafka.max.request.size: + kafka.compression.type: none + format: json + +processing_pipelines: + etl_processor: # [object] Processing Pipeline + type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + remove_fields: + output_fields: + properties: + key: value + functions: # [array of object] Function List + + - function: ASN_LOOKUP + lookup_fields: [ server_ip ] + output_fields: [ server_asn ] + parameters: + option: IP_TO_ASN + kb_name: tsg_ip_asn + + - function: ASN_LOOKUP + lookup_fields: [ client_ip ] + output_fields: [ client_asn ] + parameters: + option: IP_TO_ASN + kb_name: tsg_ip_asn + + - function: SNOWFLAKE_ID + lookup_fields: [ '' ] + output_fields: [ log_id ] + parameters: + data_center_id_num: 1 + + - function: JSON_EXTRACT + lookup_fields: [ device_tag ] + output_fields: [ data_center ] + filter: + parameters: + value_expression: $.tags[?(@.tag=='data_center')][0].value + + - function: JSON_EXTRACT + lookup_fields: [ device_tag ] + output_fields: [ device_group ] + filter: + parameters: + value_expression: $.tags[?(@.tag=='device_group')][0].value + + - function: CURRENT_UNIX_TIMESTAMP + output_fields: [ processing_time ] + parameters: + precision: seconds + + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [ __timestamp ] + output_fields: [ recv_time ] + parameters: + precision: seconds + + - function: EVAL + output_fields: [ ingestion_time ] + parameters: + value_expression: 'recv_time' + + - function: DOMAIN + lookup_fields: [ http_host,ssl_sni,dtls_sni,quic_sni ] + output_fields: [ server_domain ] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + + - function: BASE64_DECODE_TO_STRING + output_fields: [mail_subject] + parameters: + value_field: mail_subject + charset_field: mail_subject_charset + + - function: BASE64_DECODE_TO_STRING + output_fields: [mail_attachment_name] + parameters: + value_field: mail_attachment_name + charset_field: mail_attachment_name_charset + + - function: PATH_COMBINE + lookup_fields: [ rtp_pcap_path ] + output_fields: [ rtp_pcap_path ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path ] + + - function: PATH_COMBINE + lookup_fields: [ http_request_body ] + output_fields: [ http_request_body ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_request_body ] + + - function: PATH_COMBINE + lookup_fields: [ http_response_body ] + output_fields: [ http_response_body ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_response_body ] + + - function: PATH_COMBINE + lookup_fields: [ mail_eml_file ] + output_fields: [ mail_eml_file ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file ] + + - function: PATH_COMBINE + lookup_fields: [ packet_capture_file ] + output_fields: [ packet_capture_file ] + parameters: + path: [ props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file ] + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: {{ clickhouse_servers }} + table: tsg_galaxy_v3.voip_record_local + batch.size: 100000 + batch.interval: 30s + connection.user: default + connection.password: galaxy2019 + + +application: + + env: # [object] Environment Variables + name: etl_voip_record_kafka_to_clickhouse # [string] Job Name + pipeline: + object-reuse: true # [boolean] Object Reuse, default is false + topology: + - name: kafka_source + downstream: [etl_processor] + - name: etl_processor + downstream: [clickhouse_sink] + - name: clickhouse_sink + + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/dos_event_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/dos_event_kafka_to_clickhouse.sh new file mode 100644 index 0000000..bd7d35c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/dos_event_kafka_to_clickhouse.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export PARALLELISM="{{ flink.dos_event_kafka_to_clickhouse['parallelism'] }}" +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.dos_event_kafka_to_clickhouse | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_proxy_event_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_proxy_event_kafka_to_clickhouse.sh new file mode 100644 index 0000000..9b067c8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_proxy_event_kafka_to_clickhouse.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export PARALLELISM="{{ flink.etl_proxy_event_kafka_to_clickhouse['parallelism'] }}" + +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.etl_proxy_event_kafka_to_clickhouse | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_session_record_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_session_record_kafka_to_clickhouse.sh new file mode 100644 index 0000000..0f46e7f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_session_record_kafka_to_clickhouse.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export PARALLELISM="{{ flink.etl_session_record_kafka_to_clickhouse['parallelism'] }}" + +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.etl_session_record_kafka_to_clickhouse | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_transaction_record_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_transaction_record_kafka_to_clickhouse.sh new file mode 100644 index 0000000..bfe455a --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_transaction_record_kafka_to_clickhouse.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export PARALLELISM="{{ flink.etl_transaction_record_kafka_to_clickhouse['parallelism'] }}" + +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.etl_transaction_record_kafka_to_clickhouse | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_voip_record_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_voip_record_kafka_to_clickhouse.sh new file mode 100644 index 0000000..d7815b2 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_voip_record_kafka_to_clickhouse.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export PARALLELISM="{{ flink.etl_voip_record_kafka_to_clickhouse['parallelism'] }}" + +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.etl_voip_record_kafka_to_clickhouse | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/start.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/start.sh new file mode 100644 index 0000000..e119510 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/start.sh @@ -0,0 +1,64 @@ +#!/bin/bash +source /etc/profile + +# 任务jar所在目录 +BASE_DIR="$(dirname "$(pwd)")" + +SESSION_CLUSTER="Flink session cluster" + +ENV_DIR=$BASE_DIR/env/${TASK_NAME}.sh + +CONFIG_PATH=${1:-$BASE_DIR/config/} + +for file in $(ls "${CONFIG_PATH}"); do + if [ -f "$CONFIG_PATH/$file" ]; then + TASK_NAME=${file} + if [ -f "$BASE_DIR/env/${TASK_NAME}.sh" ]; then + # Promote all variable declarations to environment (exported) variables + set -a + . "$BASE_DIR/env/${TASK_NAME}.sh" + set +a + else + exit 1 + fi + + case ${TASK_MODE} in + yarn-per-job) + HAS_TASK=$(yarn application -list | grep "$TASK_NAME" | wc -l) + if [ ${HAS_TASK} -eq "0" ]; then + $GROOT_HOME/bin/start.sh --target $TASK_MODE ${FLINK_JOB_OPTS} \ + -Dyarn.application.name=$TASK_NAME \ + -Dmetrics.reporter.promgateway.jobName=$TASK_NAME \ + -p $PARALLELISM -d -c $CONFIG_PATH/$TASK_NAME + + echo -e "\033[32;1m ${TASK_NAME} job started. \033[0m" + else + echo -e "\033[31;1m ${TASK_NAME} job started,Do not submit tasks repeatedly! \033[0m" + fi + ;; + yarn-session) + HAS_SESSION=$(yarn application -list | grep "${SESSION_CLUSTER}" | wc -l) + if [ ${HAS_SESSION} -ne '0' ]; then + SESSION_ID=$(yarn application -list | grep "${SESSION_CLUSTER}" | awk '{print $1}') + HAS_TASK=$(flink list -t yarn-session -Dyarn.application.id="${SESSION_ID}" | grep "${TASK_NAME}" | grep -v flink | wc -l) + if [ -n "${SESSION_ID}" ] && [ ${HAS_TASK} -eq "0" ]; then + $GROOT_HOME/bin/start.sh --target $TASK_MODE -Dyarn.application.id=${SESSION_ID} \ + -p ${PARALLELISM} -d -c ${CONFIG_PATH}/${TASK_NAME} -n ${TASK_NAME} + + echo -e "\033[32;1m ${TASK_NAME} job started. \033[0m" + else + echo -e "\033[31;1m ${TASK_NAME} job started,Do not submit tasks repeatedly! \033[0m" + fi + else + echo -e "\033[31;1m Check whether the $SESSION_CLUSTER container is not found in the Yarn cluster. \033[0m" + fi + ;; + *) + echo "Invalid DEPLOYMENT_MODE. Choose from: standalone, standalone-docker, yarn-per-job, yarn-session" + exit 1 + ;; + esac + fi +done + + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/stop.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/stop.sh new file mode 100644 index 0000000..434429c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/stop.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# flink任务停止脚本 +source /etc/profile + +# yarn-per-job, yarn-session + +SESSION_CLUSTER="Flink session cluster" + +CONFIG_PATH=${1:-$BASE_DIR/config/} + +function stopJob() { + TASK_NAME=$1 + + case $TASK_MODE in + yarn-per-job) + HAS_TASK=$(yarn application -list | grep "$TASK_NAME" | wc -l) + if [ $HAS_TASK -ne "0" ]; then + SESSION_ID=$(yarn application -list | grep "$TASK_NAME" | awk '{print $1}') + yarn application -kill $SESSION_ID + fi + ;; + yarn-session) + SESSION_ID=$(yarn application -list | grep "$SESSION_CLUSTER" | awk '{print $1}') + HAS_TASK=$(flink list -t yarn-session -Dyarn.application.id="$SESSION_ID" | grep "$TASK_NAME" | grep -v flink | wc -l) + if [ -n $SESSION_ID ] && [ $HAS_TASK -ne "0" ]; then + JOB_ID=$(flink list -t yarn-session -Dyarn.application.id="$SESSION_ID" | grep -v flink | grep "$TASK_NAME" | awk '{print $4}') + flink cancel -t yarn-session -Dyarn.application.id=$SESSION_ID $JOB_ID + fi + ;; + *) + echo "Invalid DEPLOYMENT_MODE. Choose from: standalone, yarn-per-job, yarn-session" + exit 1 + ;; + esac +} + +#如果是文件则停止一个任务 +if [ -f "$1" ]; then + stopJob $1 +fi + +#如果是文件夹则遍历停止 +if [ -d "$1" ]; then + for file in $(ls "$1"); do + stopJob $file + done +fi + + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/config/correlation_sip_rtp_session.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/config/correlation_sip_rtp_session.j2 new file mode 100644 index 0000000..0007cd9 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/config/correlation_sip_rtp_session.j2 @@ -0,0 +1,46 @@ +#Source configurations +source.kafka.topic=VOIP-RECORD + +source.kafka.props.bootstrap.servers={{ kafka_source_servers }} + +source.kafka.props.group.id=sip-rtp-correlation-20230821-1 + +source.kafka.props.security.protocol=SASL_PLAINTEXT + +source.kafka.props.sasl.mechanism=PLAIN + +source.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + + +#Sink configurations +sink.kafka.topic=VOIP-CONVERSATION-RECORD + +sink.kafka.props.bootstrap.servers={{ kafka_sink_servers }} + +sink.kafka.props.security.protocol=SASL_PLAINTEXT + +sink.kafka.props.sasl.mechanism=PLAIN + +sink.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + + +#Correlation configurations +sip.state.clear.interval.minutes=1 + +rtp.state.clear.interval.minutes=5 + +#Error log configurations +error.records.output.enable=true + +error.sink.kafka.topic=VOIP-CONVERSATION-RECORD + +error.sink.kafka.props.bootstrap.servers=192.168.45.102:9094 + +error.sink.kafka.props.security.protocol=SASL_PLAINTEXT + +error.sink.kafka.props.sasl.mechanism=PLAIN + +error.sink.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + +include.intranet.ip=true + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/env/correlation_sip_rtp_session.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/env/correlation_sip_rtp_session.sh new file mode 100644 index 0000000..ea91f63 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/env/correlation_sip_rtp_session.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.zdjizhi.flink.voip.CorrelateApp" + +export PARALLELISM="{{ flink.correlation_sip_rtp_session['parallelism'] }}" +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +{% for item in flink.correlation_sip_rtp_session | dict2items %} +{% if item.key != 'parallelism' %} +-D{{ item.key }}={{ item.value }} +{% endif %} +{% endfor %} +" diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/start.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/start.sh new file mode 100644 index 0000000..44b015c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/start.sh @@ -0,0 +1,68 @@ +#!/bin/bash +source /etc/profile + +# 任务jar所在目录 +BASE_DIR="$(dirname "$(pwd)")" + +SESSION_CLUSTER="Flink session cluster" + +#Check the number of jar +if [ `ls ${BASE_DIR}/lib | grep jar | wc -l` -eq '1' ];then + JAR_NAME=`ls ${BASE_DIR}/lib | grep jar` +else + echo -e "\033[31;1m There are too many jars in the current directory. Keep the latest files! \033[0m" + exit 1 +fi + +JAR_DIR=${BASE_DIR}/lib/${JAR_NAME} + +CONFIG_PATH=${1:-$BASE_DIR/config/} + +for file in $(ls "${CONFIG_PATH}"); do +if [ -f "$CONFIG_PATH/$file" ]; then + TASK_NAME=${file} + #Load job env + if [ -f "$BASE_DIR/env/${TASK_NAME}.sh" ]; then + # Promote all variable declarations to environment (exported) variables + set -a + . "$BASE_DIR/env/${TASK_NAME}.sh" + set +a + else + exit 1 + fi + + case ${TASK_MODE} in + yarn-per-job) + HAS_TASK=$(yarn application -list | grep "$TASK_NAME" | wc -l) + if [ ${HAS_TASK} -eq "0" ]; then + flink run -t yarn-per-job ${FLINK_JOB_OPTS} -Dyarn.application.name=${TASK_NAME} -Dmetrics.reporter.promgateway.jobName=${TASK_NAME} \ + -p ${PARALLELISM} -d -c ${MAIN_CLASS} ${JAR_DIR} ${CONFIG_PATH}/${TASK_NAME} + echo -e "\033[32;1m ${TASK_NAME} job started. \033[0m" + else + echo -e "\033[31;1m ${TASK_NAME} job started,Do not submit tasks repeatedly! \033[0m" + fi + ;; + yarn-session) + HAS_SESSION=$(yarn application -list | grep "${SESSION_CLUSTER}" | wc -l) + if [ ${HAS_SESSION} -ne '0' ]; then + SESSION_ID=$(yarn application -list | grep "${SESSION_CLUSTER}" | awk '{print $1}') + HAS_TASK=$(flink list -t yarn-session -Dyarn.application.id="${SESSION_ID}" | grep "${TASK_NAME}" | grep -v flink | wc -l) + if [ -n "${SESSION_ID}" ] && [ ${HAS_TASK} -eq "0" ]; then + flink run -t yarn-session -Dyarn.application.id=${SESSION_ID} \ + -p ${PARALLELISM} -d -c ${MAIN_CLASS} ${JAR_DIR} ${CONFIG_PATH}/${TASK_NAME} + echo -e "\033[32;1m ${TASK_NAME} job started. \033[0m" + else + echo -e "\033[31;1m ${TASK_NAME} job started,Do not submit tasks repeatedly! \033[0m" + fi + else + echo -e "\033[31;1m Check whether the $SESSION_CLUSTER container is not found in the Yarn cluster. \033[0m" + fi + ;; + *) + echo "Invalid DEPLOYMENT_MODE. Choose from: standalone, standalone-docker, yarn-per-job, yarn-session" + exit 1 + ;; + esac +fi +done + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/stop.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/stop.sh new file mode 100644 index 0000000..86492f7 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/stop.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# flink任务停止脚本 +source /etc/profile + +BASE_DIR="$(dirname "$(pwd)")" +SESSION_CLUSTER="Flink session cluster" + +function stopJob(){ +TASK_NAME=$1 + if [ -f "$BASE_DIR/env/${TASK_NAME}.sh" ]; then + # Promote all variable declarations to environment (exported) variables + set -a + . "$BASE_DIR/env/${TASK_NAME}.sh" + set +a + else + exit 1 + fi + +case ${TASK_MODE} in + yarn-per-job) + HAS_TASK=$(yarn application -list | grep "${TASK_NAME}" | wc -l) + if [ ${HAS_TASK} -ne "0" ]; then + SESSION_ID=$(yarn application -list | grep "${TASK_NAME}" | awk '{print $1}') + yarn application -kill ${SESSION_ID} + fi + ;; + yarn-session) + SESSION_ID=$(yarn application -list | grep "$SESSION_CLUSTER" | awk '{print $1}') + HAS_TASK=$(flink list -t yarn-session -Dyarn.application.id="${SESSION_ID}" | grep "${TASK_NAME}" | grep -v flink | wc -l) + if [ -n ${SESSION_ID} ] && [ ${HAS_TASK} -ne "0" ]; then + JOB_ID=$(flink list -t yarn-session -Dyarn.application.id="${SESSION_ID}" | grep -v flink | grep "${TASK_NAME}" | awk '{print $4}') + flink cancel -t yarn-session -Dyarn.application.id=${SESSION_ID} ${JOB_ID} + fi + ;; + *) + echo "Invalid DEPLOYMENT_MODE. Choose from: standalone, yarn-per-job, yarn-session" + exit 1 + ;; +esac +} + +CONFIG_PATH=${1:-$BASE_DIR/config} + +#如果是文件则停止一个任务 +if [ -f "$CONFIG_PATH" ]; then + stopJob $CONFIG_PATH +fi + +#如果是文件夹则遍历停止 +if [ -d "$CONFIG_PATH" ];then + for file in $(ls "$CONFIG_PATH"); do + stopJob $file + done +fi + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/config/agg_session_record_topn.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/config/agg_session_record_topn.j2 new file mode 100644 index 0000000..df040ee --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/config/agg_session_record_topn.j2 @@ -0,0 +1,69 @@ +#--------------------------------Kafka消费组信息------------------------------# +#kafka的地址信息 +kafka.consumer.broker={{ kafka_source_servers }} + +#kafka 接收数据topic +kafka.consumer.topic=SESSION-RECORD + +#消费组名称 +kafka.consumer.group.id=topn-metrics-job-20230501 + +#--------------------------------Kafka生产者信息------------------------------# +#kafka的地址信息 +kafka.producer.broker={{ kafka_sink_servers }} + +#数据输出topic +kafka.producer.topic=TRAFFIC-TOP-METRIC + +#--------------------------------topology配置------------------------------# +#任务名称 +job.name=agg_session_record_topn + +#任务并行度 +task.parallelism={{ flink.agg_session_record_topn['parallelism'] }} + +#source并行度 +kafka.consumer.parallelism={{ flink.agg_session_record_topn['parallelism'] }} + +#二次排序并行度 +orderby.parallelism={{ flink.agg_session_record_topn['parallelism'] }} + +#入库并行度，通常等于orderby.parallelism +sink.parallelism={{ flink.agg_session_record_topn['parallelism'] }} + +#top结果限制 +top.limit=1000 + +#窗口延迟等待时间单位秒 +watermark.time=90 + +#滑动窗口总时间单位分钟 +window.time.minute=1 + +#--------------------------------Kafka Consumer Config------------------------------# +#kafka消费者是否开启安全验证 0不开启 1SSL 2 SASL +kafka.consumer.security=2 + +#kafka SASL验证用户名 +kafka.consumer.user=admin + +#kafka SASL及SSL验证密码 +kafka.consumer.pin=galaxy2019 + +#1SSL需要 +tools.consumer.library={{ deploy_dir }}/topology/data/ + +#--------------------------------Kafka Producer Config------------------------------# +#kafka消费者是否开启安全验证 0不开启 1SSL 2 SASL +kafka.producer.security=2 + +#kafka SASL验证用户名 +kafka.producer.user=admin + +#kafka SASL及SSL验证密码 +kafka.producer.pin=galaxy2019 + +#1SSL需要 +tools.producer.library={{ deploy_dir }}/topology/data/ + + diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/env/agg_session_record_topn.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/env/agg_session_record_topn.sh new file mode 100644 index 0000000..939c4e0 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/env/agg_session_record_topn.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +export MAIN_CLASS="com.galaxy.tsg.Toptask" + +export PARALLELISM="{{ flink.agg_session_record_topn['parallelism'] }}" +export TASK_MODE="{{ task_mode }}" + +export FLINK_JOB_OPTS=" +-Djobmanager.memory.process.size=1024m +-Dtaskmanager.memory.process.size={{ flink.agg_session_record_topn['taskmanager.memory.process.size'] }} +-Dtaskmanager.numberOfTaskSlots={{ flink.agg_session_record_topn['taskmanager.numberOfTaskSlots'] }} +-Dtaskmanager.memory.framework.off-heap.size={{ flink.agg_session_record_topn['taskmanager.memory.framework.off-heap.size'] }} +" diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/upload_knowlodge.sh.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/upload_knowlodge.sh.j2 new file mode 100644 index 0000000..9049910 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/templates/upload_knowlodge.sh.j2 @@ -0,0 +1,6 @@ +curl --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base' --form 'file=@"{{ deploy_dir }}/flink/topology/data/asn_builtin.mmdb"' --form 'kb_id=f9f6bc91-2142-4673-8249-e097c00fe1ea' --form 'name=asn_builtin' --form 'format=mmdb' --form 'is_valid=1' + +curl --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base' --form 'file=@"{{ deploy_dir }}/flink/topology/data/ip_builtin.mmdb"' --form 'kb_id=64af7077-eb9b-4b8f-80cf-2ceebc89bea9' --form 'name=ip_builtin' --form 'format=mmdb' --form 'is_valid=1' + +curl --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base' --form 'file=@"{{ deploy_dir }}/flink/topology/data/ip_user_defined.mmdb"' --form 'kb_id=004390bc-3135-4a6f-a492-3662ecb9e289' --form 'name=ip_user_defined' --form 'format=mmdb' --form 'is_valid=1' + diff --git a/tsg-olap/parcels/roles/initialization/yarn/vars/main.yml b/tsg-olap/parcels/roles/initialization/yarn/vars/main.yml new file mode 100644 index 0000000..5f59e9a --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/yarn/vars/main.yml @@ -0,0 +1,21 @@ +task_mode: "{{ 'yarn-per-job' if groups['yarn'] |length > 1 else 'yarn-session' }}" + +kafka_source_servers: "{% for host in groups['kafka'] %}{{ host }}:9094{% if not loop.last %},{% endif %}{% endfor %}" + +kafka_sink_servers: "{{ kafka_source_servers }}" + +zookeeper_servers: "{% for host in groups['kafka'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}" + +clickhouse_servers: "{% if groups['clickhouse'] | length < 3 %}{{ groups['clickhouse'][0] }}:9001{% else %}{% for host in groups['clickhouse'][2:] %}{{ host }}:9001{% if not loop.last %},{% endif %}{% endfor %}{% endif %}" + +#groot版本 +groot_stream_version: groot-stream-1.0 + +#Jdk版本 +java_version: 1.8.0_73 + + +backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/flink" + +backup_items: + - topology