diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml deleted file mode 100644 index dfb0b32..0000000 --- a/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml +++ /dev/null @@ -1,11 +0,0 @@ -- name: Set arangodb check variable - set_fact: - pin: "root:{{ arangodb_default_pin }}" - -- name: Check if collection exists in ArangoDB - uri: - url: "http://{{ groups.arangodb[0] }}:8529/_db/{{ arangodb_database }}/_api/collection/{{ item }}" - method: GET - headers: - Authorization: Basic {{ pin | b64encode }} - loop: "{{ arangodb_collections }}" diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml deleted file mode 100644 index 920b6d4..0000000 --- a/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml +++ /dev/null @@ -1,21 +0,0 @@ -- name: Creating directory - file: - state: directory - path: '{{ deploy_dir }}/{{ container_name }}/init' - -- name: Copying init files - template: - src: "create_database.sh.j2" - dest: "{{ deploy_dir }}/{{ container_name }}/init/create_database.sh" - mode: 0755 - force: true - -- name: Creating database and collection - shell: 'cd {{ deploy_dir }}/{{ container_name }}/init && sh create_database.sh' - register: response - until: response.rc == 0 - retries: 3 - delay: 3 - failed_when: response.rc != 0 - run_once: true - diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml deleted file mode 100644 index 8894d83..0000000 --- a/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -- include: init.yml - when: operation == 'install' -- include: check.yml diff --git a/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 b/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 deleted file mode 100644 index 29657da..0000000 --- a/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -source /etc/profile - -arangodb_host={{ groups.arangodb[0] | default('localhost') }} -username=root -pin="{{ arangodb_default_pin }}" -database="{{ arangodb_database }}" - -function create_database() { - # 检查数据库是否存在 - sleep 2 - check_database=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" http://$arangodb_host:8529/_db/$database/_api/database/current) - if [[ $check_database -eq 200 ]]; then - echo "Database $database already exists." - else - # 创建数据库 - local response_code=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" -X POST -H "accept: application/json" -d "{\"name\": \"$database\"}" http://$arangodb_host:8529/_api/database) - if [[ $response_code -eq 201 ]]; then - echo "Database $database created successfully." - else - echo "Failed to create database $database." - exit 1 - fi - fi - sleep 2 - -} - -function create_collection() { - local collection_name=$1 - local collection_type=$2 - sleep 2 - # 检查文档是否存在 - check_collection=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" http://$arangodb_host:8529/_db/$database/_api/collection/$collection_name) - if [[ $check_collection -eq 200 ]]; then - echo "Collection $collection_name already exists." - else - # 创建文档$collection_name - local response_code=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" -X POST -H 'accept: application/json' -d "{\"name\": \"$collection_name\",\"type\": $collection_type }" http://$arangodb_host:8529/_db/$database/_api/collection?isSystem=false) - if [[ $response_code -eq 201 ]]; then - echo "Collection $collection_name created successfully." - else - echo "Failed to create collection $collection_name." - exit 1 - fi - fi - sleep 2 -} - -# 2: document 3: edges -create_database -create_collection IP 2 -create_collection R_LOCATE_FQDN2IP 3 - diff --git a/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml b/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml deleted file mode 100644 index fa8b9d4..0000000 --- a/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml +++ /dev/null @@ -1,9 +0,0 @@ -container_name: arangodb - -# 数据库 -arangodb_database: 'tsg_galaxy_v3' - -# 库表 -arangodb_collections: - - IP - - R_LOCATE_FQDN2IP diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql deleted file mode 100644 index 7eb3923..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql +++ /dev/null @@ -1,4174 +0,0 @@ -create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; -create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query; - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster ( - vsys_id Int32, - recv_time Int64, - log_id UInt64, - profile_id Int64, - start_time Int64, - end_time Int64, - attack_type String, - severity String, - conditions String, - destination_ip String, - destination_country String, - source_ip_list String, - source_country_list String, - session_rate Int64, - packet_rate Int64, - bit_rate Int64 -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,destination_ip,recv_time,log_id); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster ( - vsys_id Int32, - recv_time Int64, - log_id UInt64, - profile_id Int64, - start_time Int64, - end_time Int64, - attack_type String, - severity String, - conditions String, - destination_ip String, - destination_country String, - source_ip_list String, - source_country_list String, - session_rate Int64, - packet_rate Int64, - bit_rate Int64 -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_query ( - vsys_id Int32, - recv_time Int64, - log_id UInt64, - profile_id Int64, - start_time Int64, - end_time Int64, - attack_type String, - severity String, - conditions String, - destination_ip String, - destination_country String, - source_ip_list String, - source_country_list String, - session_rate Int64, - packet_rate Int64, - bit_rate Int64 -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster ( - log_id UInt64, - recv_time Int64, - vsys_id Int64, - assessment_date Int64, - lot_number String, - file_name String, - assessment_file String, - assessment_type String, - features String, - size Int64, - file_checksum_sha String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,recv_time,log_id); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query ( - log_id UInt64, - recv_time Int64, - vsys_id Int64, - assessment_date Int64, - lot_number String, - file_name String, - assessment_file String, - assessment_type String, - features String, - size Int64, - file_checksum_sha String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster ( - log_id UInt64, - recv_time Int64, - vsys_id Int64, - assessment_date Int64, - lot_number String, - file_name String, - assessment_file String, - assessment_type String, - features String, - size Int64, - file_checksum_sha String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); - - - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -address_type Int32, -vsys_id Int32, -client_ip String, -client_port Int32, -server_ip String, -server_port Int32, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,session_id,recv_time); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -address_type Int32, -vsys_id Int32, -client_ip String, -client_port Int32, -server_ip String, -server_port Int32, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -address_type Int32, -vsys_id Int32, -client_ip String, -client_port Int32, -server_ip String, -server_port Int32, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_eml_file String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); - - -alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; -alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); - - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -doh_url String, -doh_host String, -doh_request_line String, -doh_response_line String, -doh_cookie String, -doh_referer String, -doh_user_agent String, -doh_content_length String, -doh_content_type String, -doh_set_cookie String, -doh_version String, -doh_message_id Int64, -doh_qr Nullable(Int64), -doh_opcode Nullable(Int64), -doh_aa Nullable(Int64), -doh_tc Nullable(Int64), -doh_rd Nullable(Int64), -doh_ra Nullable(Int64), -doh_rcode Nullable(Int64), -doh_qdcount Nullable(Int64), -doh_ancount Nullable(Int64), -doh_nscount Nullable(Int64), -doh_arcount Nullable(Int64), -doh_qname String, -doh_qtype Nullable(Int64), -doh_qclass Nullable(Int64), -doh_cname String, -doh_sub Nullable(Int64), -doh_rr String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -doh_url String, -doh_host String, -doh_request_line String, -doh_response_line String, -doh_cookie String, -doh_referer String, -doh_user_agent String, -doh_content_length String, -doh_content_type String, -doh_set_cookie String, -doh_version String, -doh_message_id Int64, -doh_qr Nullable(Int64), -doh_opcode Nullable(Int64), -doh_aa Nullable(Int64), -doh_tc Nullable(Int64), -doh_rd Nullable(Int64), -doh_ra Nullable(Int64), -doh_rcode Nullable(Int64), -doh_qdcount Nullable(Int64), -doh_ancount Nullable(Int64), -doh_nscount Nullable(Int64), -doh_arcount Nullable(Int64), -doh_qname String, -doh_qtype Nullable(Int64), -doh_qclass Nullable(Int64), -doh_cname String, -doh_sub Nullable(Int64), -doh_rr String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -doh_url String, -doh_host String, -doh_request_line String, -doh_response_line String, -doh_cookie String, -doh_referer String, -doh_user_agent String, -doh_content_length String, -doh_content_type String, -doh_set_cookie String, -doh_version String, -doh_message_id Int64, -doh_qr Nullable(Int64), -doh_opcode Nullable(Int64), -doh_aa Nullable(Int64), -doh_tc Nullable(Int64), -doh_rd Nullable(Int64), -doh_ra Nullable(Int64), -doh_rcode Nullable(Int64), -doh_qdcount Nullable(Int64), -doh_ancount Nullable(Int64), -doh_nscount Nullable(Int64), -doh_arcount Nullable(Int64), -doh_qname String, -doh_qtype Nullable(Int64), -doh_qclass Nullable(Int64), -doh_cname String, -doh_sub Nullable(Int64), -doh_rr String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -tunnels String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); - - -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster -TO tsg_galaxy_v3.security_event_local -( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - tunnels String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -AS -SELECT - recv_time, - log_id, - decoded_as, - session_id, - start_timestamp_ms, - end_timestamp_ms, - duration_ms, - tcp_handshake_latency_ms, - ingestion_time, - processing_time, - -- insert_time, - device_id, - out_link_id, - in_link_id, - device_tag, - data_center, - device_group, - sled_ip, - address_type, - vsys_id, - t_vsys_id, - flags, - flags_identify_info, - security_rule_list, - security_action, - monitor_rule_list, - shaping_rule_list, - proxy_rule_list, - statistics_rule_list, - sc_rule_list, - sc_rsp_raw, - sc_rsp_decrypted, - proxy_action, - proxy_pinning_status, - proxy_intercept_status, - proxy_passthrough_reason, - proxy_client_side_latency_ms, - proxy_server_side_latency_ms, - proxy_client_side_version, - proxy_server_side_version, - proxy_cert_verify, - proxy_intercept_error, - monitor_mirrored_pkts, - monitor_mirrored_bytes, - client_ip, - client_port, - client_os_desc, - client_geolocation, - client_asn, - subscriber_id, - imei, - imsi, - phone_number, - apn, - server_ip, - server_port, - server_os_desc, - server_geolocation, - server_asn, - server_fqdn, - server_domain, - app_transition, - app, - app_debug_info, - app_content, - fqdn_category_list, - decoded_path, - dns_message_id, - dns_qr, - dns_opcode, - dns_aa, - dns_tc, - dns_rd, - dns_ra, - dns_rcode, - dns_qdcount, - dns_ancount, - dns_nscount, - dns_arcount, - dns_qname, - dns_qtype, - dns_qclass, - dns_cname, - dns_sub, - dns_rr, - dns_response_latency_ms, - http_url, - http_host, - http_request_line, - http_response_line, - http_request_body, - http_response_body, - http_proxy_flag, - http_sequence, - http_cookie, - http_referer, - http_user_agent, - http_request_content_length, - http_request_content_type, - http_response_content_length, - http_response_content_type, - http_set_cookie, - http_version, - http_status_code, - http_response_latency_ms, - http_session_duration_ms, - http_action_file_size, - ssl_version, - ssl_sni, - ssl_san, - ssl_cn, - ssl_handshake_latency_ms, - ssl_ja3_hash, - ssl_ja3s_hash, - ssl_cert_issuer, - ssl_cert_subject, - ssl_esni_flag, - ssl_ech_flag, - dtls_cookie, - dtls_version, - dtls_sni, - dtls_san, - dtls_cn, - dtls_handshake_latency_ms, - dtls_ja3_fingerprint, - dtls_ja3_hash, - dtls_cert_issuer, - dtls_cert_subject, - mail_protocol_type, - mail_account, - mail_from_cmd, - mail_to_cmd, - mail_from, - mail_password, - mail_to, - mail_cc, - mail_bcc, - mail_subject, - mail_subject_charset, - mail_attachment_name, - mail_attachment_name_charset, - mail_eml_file, - ftp_account, - ftp_url, - ftp_link_type, - quic_version, - quic_sni, - quic_user_agent, - rdp_cookie, - rdp_security_protocol, - rdp_client_channels, - rdp_keyboard_layout, - rdp_client_version, - rdp_client_name, - rdp_client_product_id, - rdp_desktop_width, - rdp_desktop_height, - rdp_requested_color_depth, - rdp_certificate_type, - rdp_certificate_count, - rdp_certificate_permanent, - rdp_encryption_level, - rdp_encryption_method, - ssh_version, - ssh_auth_success, - ssh_client_version, - ssh_server_version, - ssh_cipher_alg, - ssh_mac_alg, - ssh_compression_alg, - ssh_kex_alg, - ssh_host_key_alg, - ssh_host_key, - ssh_hassh, - sip_call_id, - sip_originator_description, - sip_responder_description, - sip_user_agent, - sip_server, - sip_originator_sdp_connect_ip, - sip_originator_sdp_media_port, - sip_originator_sdp_media_type, - sip_originator_sdp_content, - sip_responder_sdp_connect_ip, - sip_responder_sdp_media_port, - sip_responder_sdp_media_type, - sip_responder_sdp_content, - sip_duration_s, - sip_bye, - rtp_payload_type_c2s, - rtp_payload_type_s2c, - rtp_pcap_path, - rtp_originator_dir, - stratum_cryptocurrency, - stratum_mining_pools, - stratum_mining_program, - stratum_mining_subscribe, - sent_pkts, - received_pkts, - sent_bytes, - received_bytes, - tcp_c2s_ip_fragments, - tcp_s2c_ip_fragments, - tcp_c2s_lost_bytes, - tcp_s2c_lost_bytes, - tcp_c2s_o3_pkts, - tcp_s2c_o3_pkts, - tcp_c2s_rtx_pkts, - tcp_s2c_rtx_pkts, - tcp_c2s_rtx_bytes, - tcp_s2c_rtx_bytes, - tcp_rtt_ms, - tcp_client_isn, - tcp_server_isn, - packet_capture_file, - in_src_mac, - out_src_mac, - in_dest_mac, - out_dest_mac, - tunnels, - dup_traffic_flag, - tunnel_endpoint_a_desc, - tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.session_record_local -WHERE empty(security_rule_list) = 0 -; - --- tsg_galaxy_v3.monitor_event_materialized_view -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster -TO tsg_galaxy_v3.monitor_event_local -( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - tunnels String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -AS -SELECT - recv_time, - log_id, - decoded_as, - session_id, - start_timestamp_ms, - end_timestamp_ms, - duration_ms, - tcp_handshake_latency_ms, - ingestion_time, - processing_time, - -- insert_time, - device_id, - out_link_id, - in_link_id, - device_tag, - data_center, - device_group, - sled_ip, - address_type, - vsys_id, - t_vsys_id, - flags, - flags_identify_info, - security_rule_list, - security_action, - monitor_rule_list, - shaping_rule_list, - proxy_rule_list, - statistics_rule_list, - sc_rule_list, - sc_rsp_raw, - sc_rsp_decrypted, - proxy_action, - proxy_pinning_status, - proxy_intercept_status, - proxy_passthrough_reason, - proxy_client_side_latency_ms, - proxy_server_side_latency_ms, - proxy_client_side_version, - proxy_server_side_version, - proxy_cert_verify, - proxy_intercept_error, - monitor_mirrored_pkts, - monitor_mirrored_bytes, - client_ip, - client_port, - client_os_desc, - client_geolocation, - client_asn, - subscriber_id, - imei, - imsi, - phone_number, - apn, - server_ip, - server_port, - server_os_desc, - server_geolocation, - server_asn, - server_fqdn, - server_domain, - app_transition, - app, - app_debug_info, - app_content, - fqdn_category_list, - decoded_path, - dns_message_id, - dns_qr, - dns_opcode, - dns_aa, - dns_tc, - dns_rd, - dns_ra, - dns_rcode, - dns_qdcount, - dns_ancount, - dns_nscount, - dns_arcount, - dns_qname, - dns_qtype, - dns_qclass, - dns_cname, - dns_sub, - dns_rr, - dns_response_latency_ms, - http_url, - http_host, - http_request_line, - http_response_line, - http_request_body, - http_response_body, - http_proxy_flag, - http_sequence, - http_cookie, - http_referer, - http_user_agent, - http_request_content_length, - http_request_content_type, - http_response_content_length, - http_response_content_type, - http_set_cookie, - http_version, - http_status_code, - http_response_latency_ms, - http_session_duration_ms, - http_action_file_size, - ssl_version, - ssl_sni, - ssl_san, - ssl_cn, - ssl_handshake_latency_ms, - ssl_ja3_hash, - ssl_ja3s_hash, - ssl_cert_issuer, - ssl_cert_subject, - ssl_esni_flag, - ssl_ech_flag, - dtls_cookie, - dtls_version, - dtls_sni, - dtls_san, - dtls_cn, - dtls_handshake_latency_ms, - dtls_ja3_fingerprint, - dtls_ja3_hash, - dtls_cert_issuer, - dtls_cert_subject, - mail_protocol_type, - mail_account, - mail_from_cmd, - mail_to_cmd, - mail_from, - mail_password, - mail_to, - mail_cc, - mail_bcc, - mail_subject, - mail_subject_charset, - mail_attachment_name, - mail_attachment_name_charset, - mail_eml_file, - ftp_account, - ftp_url, - ftp_link_type, - quic_version, - quic_sni, - quic_user_agent, - rdp_cookie, - rdp_security_protocol, - rdp_client_channels, - rdp_keyboard_layout, - rdp_client_version, - rdp_client_name, - rdp_client_product_id, - rdp_desktop_width, - rdp_desktop_height, - rdp_requested_color_depth, - rdp_certificate_type, - rdp_certificate_count, - rdp_certificate_permanent, - rdp_encryption_level, - rdp_encryption_method, - ssh_version, - ssh_auth_success, - ssh_client_version, - ssh_server_version, - ssh_cipher_alg, - ssh_mac_alg, - ssh_compression_alg, - ssh_kex_alg, - ssh_host_key_alg, - ssh_host_key, - ssh_hassh, - sip_call_id, - sip_originator_description, - sip_responder_description, - sip_user_agent, - sip_server, - sip_originator_sdp_connect_ip, - sip_originator_sdp_media_port, - sip_originator_sdp_media_type, - sip_originator_sdp_content, - sip_responder_sdp_connect_ip, - sip_responder_sdp_media_port, - sip_responder_sdp_media_type, - sip_responder_sdp_content, - sip_duration_s, - sip_bye, - rtp_payload_type_c2s, - rtp_payload_type_s2c, - rtp_pcap_path, - rtp_originator_dir, - stratum_cryptocurrency, - stratum_mining_pools, - stratum_mining_program, - stratum_mining_subscribe, - sent_pkts, - received_pkts, - sent_bytes, - received_bytes, - tcp_c2s_ip_fragments, - tcp_s2c_ip_fragments, - tcp_c2s_lost_bytes, - tcp_s2c_lost_bytes, - tcp_c2s_o3_pkts, - tcp_s2c_o3_pkts, - tcp_c2s_rtx_pkts, - tcp_s2c_rtx_pkts, - tcp_c2s_rtx_bytes, - tcp_s2c_rtx_bytes, - tcp_rtt_ms, - tcp_client_isn, - tcp_server_isn, - packet_capture_file, - in_src_mac, - out_src_mac, - in_dest_mac, - out_dest_mac, - tunnels, - dup_traffic_flag, - tunnel_endpoint_a_desc, - tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.session_record_local -WHERE empty(monitor_rule_list) = 0 -; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql deleted file mode 100644 index 1f45639..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql +++ /dev/null @@ -1,9 +0,0 @@ -create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); -create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); -create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); -create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); -CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); -CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); - -alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; -alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql deleted file mode 100644 index 5e7ba07..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql +++ /dev/null @@ -1,247 +0,0 @@ -set distributed_ddl_task_timeout = 180; - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster( - common_log_id UInt64, - common_recv_time Int64, - common_server_ip String, - common_client_ip String, - common_sled_ip String, - common_entrance_id Int64, - common_subscriber_id String, - common_stream_trace_id UInt64, - common_server_domain String, - http_domain String, - ssl_sni String, - common_schema_type LowCardinality(String), - common_vsys_id Int64 DEFAULT 1, - common_client_port Int64, - common_server_port Int64, - common_app_label LowCardinality(String), - common_direction Nullable(Int64), - common_data_center String, - common_device_group String -) -ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) -ORDER BY (common_server_domain,common_recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster( - common_log_id UInt64, - common_recv_time Int64, - common_server_ip String, - common_client_ip String, - common_sled_ip String, - common_entrance_id Int64, - common_subscriber_id String, - common_stream_trace_id UInt64, - common_server_domain String, - http_domain String, - ssl_sni String, - common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_client_port Int64, - common_server_port Int64, - common_app_label LowCardinality(String), - common_direction Nullable(Int64), - common_data_center String, - common_device_group String -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query( - common_log_id UInt64, - common_recv_time Int64, - common_server_ip String, - common_client_ip String, - common_sled_ip String, - common_entrance_id Int64, - common_subscriber_id String, - common_stream_trace_id UInt64, - common_server_domain String, - http_domain String, - ssl_sni String, - common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_client_port Int64, - common_server_port Int64, - common_app_label LowCardinality(String), - common_direction Nullable(Int64), - common_data_center String, - common_device_group String -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); - - -ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; -ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; -ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; - -ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; -ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; -ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; - -ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; -ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; -ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; - - -ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; -ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; -ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; - -ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; -ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; -ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; - -ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; -ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; -ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; - - -ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; -ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; -ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; - -ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; -ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; -ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; - -ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; -ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; -ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; - - -drop view IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster; -drop view IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster; -drop view IF EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster; - -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; - -drop table IF EXISTS `system`.query_log_cluster on cluster ck_query; -drop table IF EXISTS `system`.disks_cluster on cluster ck_query; -drop table IF EXISTS `system`.columns_cluster on cluster ck_query; -drop table IF EXISTS `system`.parts_cluster on cluster ck_query; -drop table IF EXISTS `system`.processes_cluster on cluster ck_query; -drop table IF EXISTS `system`.tables_cluster on cluster ck_query; - - -create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); -create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); -create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); -create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); -CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); -CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); -alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; -alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY; - diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql deleted file mode 100644 index 042b51f..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql +++ /dev/null @@ -1,110 +0,0 @@ -set distributed_ddl_task_timeout = 180; - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS http_request_body_key String after common_flags; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS http_request_body_key String after common_flags; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS http_request_body_key String after common_flags; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; - - - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; - - - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; - - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; - - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; - - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; - - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; - - - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - - - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - - - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; - - - - - - - - - - - - - - - - - diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql deleted file mode 100644 index 1d3d958..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql +++ /dev/null @@ -1,47 +0,0 @@ -set distributed_ddl_task_timeout = 180; - -alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; - - -alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - - -alter table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - - -alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - -alter table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - -alter table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - -alter table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - -alter table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - -alter table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - -alter table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; -alter table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; -alter table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; - - - - - diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql deleted file mode 100644 index 0e50ac4..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql +++ /dev/null @@ -1,6 +0,0 @@ -set distributed_ddl_task_timeout = 180; - - -ALTER table tsg_galaxy_v3.dos_event_local on cluster ck_cluster add column IF NOT EXISTS profile_id UInt64 after log_id; -ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster add column IF NOT EXISTS profile_id UInt64 after log_id; -ALTER table tsg_galaxy_v3.dos_event on cluster ck_query add column IF NOT EXISTS profile_id UInt64 after log_id; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql deleted file mode 100644 index 769c47e..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql +++ /dev/null @@ -1,11 +0,0 @@ -set distributed_ddl_task_timeout = 180; - - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; - - -ALTER TABLE tsg_galaxy_v3.proxy_event_local on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; -ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; -ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_query DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql deleted file mode 100644 index 11c06a8..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql +++ /dev/null @@ -1,159 +0,0 @@ -set distributed_ddl_task_timeout = 180; - - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; - - -ALTER TABLE tsg_galaxy_v3.proxy_event_local on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; -ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; -ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_query DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql deleted file mode 100644 index 27bf30b..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql +++ /dev/null @@ -1,97 +0,0 @@ -set distributed_ddl_task_timeout = 180; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql deleted file mode 100644 index df1ba02..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql +++ /dev/null @@ -1,158 +0,0 @@ -set distributed_ddl_task_timeout = 180; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - -ALTER table tsg_galaxy_v3.active_defence_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; -ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - -ALTER table tsg_galaxy_v3.active_defence_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; -ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql deleted file mode 100644 index 1528e0c..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql +++ /dev/null @@ -1,131 +0,0 @@ -set distributed_ddl_task_timeout = 180; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; - - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql deleted file mode 100644 index 20b7e1e..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql +++ /dev/null @@ -1,76 +0,0 @@ -set distributed_ddl_task_timeout = 180; - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; - - - -ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - -ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - -ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - -ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - -ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - -ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - -ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - -ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - -ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; -ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; - diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml deleted file mode 100644 index 9c4c8b9..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml +++ /dev/null @@ -1,24 +0,0 @@ -- name: Setting clickhouse init directory - set_fact: - init_path: '{{ deploy_dir }}/clickhouse/init/' - -- name: Creatting clickhouse init directory - file: - path: '{{ init_path }}' - state: directory - -- name: Copying Clickhouse create tables sql - copy: - src: 'files/{{ item }}' - dest: '{{ init_path }}/{{ item }}' - force: true - with_items: - - create_ck_table.sql - - system.sql - -- name: Creating tables - shell: clickhouse-client -h {{ inventory_hostname }} --port 9001 -m -u default --password {{ clickhouse_default_pin }} --multiquery < {{ init_path }}/{{ item }} - loop: - - create_ck_table.sql - - system.sql - run_once: true diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml deleted file mode 100644 index 951be61..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -- include: init.yml - when: inventory_hostname in groups['clickhouse'][:2] and operation == "install" - -- include: upgrate.yml - when: inventory_hostname in groups['clickhouse'][:2] and operation == "upgrade" diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml deleted file mode 100644 index 48401ee..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml +++ /dev/null @@ -1,25 +0,0 @@ -- name: Setting clickhouse upgrade directory - set_fact: - upgrade_path: '{{ deploy_dir }}/clickhouse/upgrade/' - -- name: Creatting clickhouse upgrade directory - file: - path: '{{ upgrade_path }}' - state: directory - -- name: Copying clickhouse upgrade tables sql - copy: - src: 'files/upgrade-sql' - dest: '{{ upgrade_path }}/' - force: true - -- name: Copying clickhouse upgrade script - template: - src: 'upgrade-clickhouse-data.sh.j2' - dest: '{{ upgrade_path }}/upgrade-clickhouse-data.sh' - mode: 0075 - force: true - -- name: Upgrade clickhouse data - shell: cd {{ upgrade_path }} && sh upgrade-clickhouse-data.sh - run_once: true diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 b/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 deleted file mode 100644 index bc9fd19..0000000 --- a/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -source /etc/profile -set -e - -# 数据库连接参数 -upgrade_dir=${1:-$(pwd)/upgrade-sql} - -# 指定当前版本和历史版本 -current_version="{{ solution_version }}" -old_version="{{ old_version }}" - -# 数据库连接参数 -db_hostname="127.0.0.1" -db_username="default" -db_password="galaxy2019" - -suffix=".sql" -# 获取更新目录中的所有SQL文件 -sql_files=$(find "${upgrade_dir}" -name "*$suffix") - -# 标记是否开始执行历史版本的标识 -start_execution=false - -# 循环处理每个SQL文件 -for file in ${sql_files}; do - # 从文件名中提取版本号 - filename=$(basename "$file") - version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.') - current_version="${current_version//./}" - old_version="${old_version//./}" - # 检查是否达到历史版本 - if [[ "${version}" -gt "${old_version}" ]]; then - start_execution=true - fi - - # 检查是否开始执行历史版本之后的SQL文件 - if [[ ${start_execution} = true ]]; then - echo "Executing SQL file: ${file}" - - # 执行SQL文件 - clickhouse-client -h 127.0.0.1 --port 9001 -m -u "${db_username}" --password "${db_password}" --multiquery <"${file}" - echo "Upgrade ${version} successfully" - fi - - # 检查是否达到当前版本 - if [[ "${version}" -ge "${current_version}" ]]; then - break - fi -done - diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/druid_segments-tsg.sql b/tsg-olap/parcels/roles/initialization/druid/files/cluster/druid_segments-tsg.sql deleted file mode 100644 index c62f59c..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/cluster/druid_segments-tsg.sql +++ /dev/null @@ -1,39 +0,0 @@ -delete FROM druid.druid_segments where dataSource='application_protocol_stat' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('application_protocol_stat_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-19T10:54:07.120Z', 'application_protocol_stat', '2023-12-19T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-19T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226170706C69636174696F6E5F70726F746F636F6C5F73746174222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31395431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F6170706C69636174696F6E5F70726F746F636F6C5F737461742F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D31395431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C70726F746F636F6C5F737461636B5F69642C6170705F6E616D652C767379735F6964222C20226D657472696373223A202273657373696F6E732C6332735F706B74732C7332635F706B74732C6332735F62797465732C7332635F62797465732C6332735F667261676D656E74732C7332635F667261676D656E74732C6332735F7463705F6C6F73745F62797465732C7332635F7463705F6C6F73745F62797465732C6332735F7463705F6F6F6F726465725F706B74732C7332635F7463705F6F6F6F726465725F706B74732C6332735F7463705F72657472616E736D69747465645F706B74732C7332635F7463705F72657472616E736D69747465645F706B74732C6332735F7463705F72657472616E736D69747465645F62797465732C7332635F7463705F72657472616E736D69747465645F62797465732C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20373739312C20226964656E746966696572223A20226170706C69636174696F6E5F70726F746F636F6C5F737461745F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31395431303A35343A30372E3132305A227D); -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('monitor_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-12T10:54:07.120Z', 'monitor_rule_hits', '2023-12-12T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-12T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226D6F6E69746F725F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31325431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F6D6F6E69746F725F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D31325431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C616374696F6E222C20226D657472696373223A20226869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333337312C20226964656E746966696572223A20226D6F6E69746F725F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31325431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='object_statistics' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('object_statistics_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-19T10:54:07.120Z', 'object_statistics', '2023-12-19T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-19T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226F626A6563745F73746174697374696373222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31395431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F6F626A6563745F737461746973746963732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D31395431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C74656D706C6174655F69642C63686172745F69642C76657273696F6E2C6F626A6563745F747970652C6F626A6563745F69642C6974656D5F6964222C20226D657472696373223A2022696E5F62797465732C6F75745F62797465732C62797465732C6E65775F696E5F73657373696F6E732C6E65775F6F75745F73657373696F6E732C73657373696F6E73222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20343631372C20226964656E746966696572223A20226F626A6563745F737461746973746963735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31395431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='proxy_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('proxy_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'proxy_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A202270726F78795F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F70726F78795F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C7375625F616374696F6E2C70696E6E696E675F7374617475732C72756C655F69642C616374696F6E2C6869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333736352C20226964656E746966696572223A202270726F78795F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('security_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-12T10:54:07.120Z', 'security_rule_hits', '2023-12-12T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-12T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A202273656375726974795F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31325431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F73656375726974795F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D31325431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C616374696F6E222C20226D657472696373223A20226869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333337312C20226964656E746966696572223A202273656375726974795F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31325431303A35343A30372E3132305A227D); - -delete FROM druid.druid_segments where dataSource='service_chaining_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('service_chaining_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'service_chaining_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022736572766963655F636861696E696E675F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F736572766963655F636861696E696E675F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C7366665F70726F66696C655F69642C73665F70726F66696C655F69642C73656E745F706B74732C73656E745F62797465732C726563765F706B74732C726563765F6279746573222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333132322C20226964656E746966696572223A2022736572766963655F636861696E696E675F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='service_function_status' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('service_function_status_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'service_function_status', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022736572766963655F66756E6374696F6E5F737461747573222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F736572766963655F66756E6374696F6E5F7374617475732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C73665F70726F66696C655F69642C73665F7374617475732C73665F6C6174656E63795F7573222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323333312C20226964656E746966696572223A2022736572766963655F66756E6374696F6E5F7374617475735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='statistics_rule' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('statistics_rule_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'statistics_rule', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022737461746973746963735F72756C65222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F737461746973746963735F72756C652F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C74656D706C6174655F69642C63686172745F69642C76657273696F6E2C636C69656E745F69705F6F626A6563742C7365727665725F69705F6F626A6563742C6671646E5F63617465676F72792C636C69656E745F69702C7365727665725F69702C7365727665725F6671646E2C7365727665725F646F6D61696E2C6170706C69636174696F6E2C696E5F62797465732C6F75745F62797465732C62797465732C6E65775F6332735F666C6F77732C6E65775F7332635F666C6F77732C6E65775F696E5F73657373696F6E732C6E65775F6F75745F73657373696F6E732C73657373696F6E732C6E65775F756E65737461626C69736865645F73657373696F6E732C73796E5F706B7473222C20226D657472696373223A202273657373696F6E5F6964656E7469666965725F736B657463682C636C69656E745F69705F736B657463682C7365727665725F69705F736B657463682C696E5F6C6174656E63795F6D735F736B657463682C6F75745F6C6174656E63795F6D735F736B657463682C6C6174656E63795F6D735F736B65746368222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20393433362C20226964656E746966696572223A2022737461746973746963735F72756C655F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='statistics_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('statistics_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'statistics_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022737461746973746963735F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F737461746973746963735F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C6869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323838302C20226964656E746966696572223A2022737461746973746963735F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='sys_storage_log' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('sys_storage_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'sys_storage_log', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20227379735F73746F726167655F6C6F67222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F7379735F73746F726167655F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226C6F675F747970652C646174615F63656E7465722C6167677265676174655F73697A652C6C6173745F73746F726167652C6D61785F73697A652C757365645F73697A65222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20313938322C20226964656E746966696572223A20227379735F73746F726167655F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_client_ips' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_client_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_client_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F636C69656E745F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F636C69656E745F6970732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022636C69656E745F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333335362C20226964656E746966696572223A2022746F705F636C69656E745F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_external_ips' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_external_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_external_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F65787465726E616C5F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F65787465726E616C5F6970732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A202265787465726E616C5F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336322C20226964656E746966696572223A2022746F705F65787465726E616C5F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_internal_ips' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_internal_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_internal_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F696E7465726E616C5F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F696E7465726E616C5F6970732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022696E7465726E616C5F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336322C20226964656E746966696572223A2022746F705F696E7465726E616C5F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_server_domains' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_domains_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_domains', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F646F6D61696E73222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F646F6D61696E732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022646F6D61696E2C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333334372C20226964656E746966696572223A2022746F705F7365727665725F646F6D61696E735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_server_fqdns' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_fqdns_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_fqdns', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F6671646E73222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F6671646E732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226671646E2C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333334312C20226964656E746966696572223A2022746F705F7365727665725F6671646E735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_server_ips' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F6970732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20227365727665725F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333335362C20226964656E746966696572223A2022746F705F7365727665725F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_subscribers' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_subscribers_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_subscribers', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7375627363726962657273222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F73756273637269626572732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022737562736372696265725F69642C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336382C20226964656E746966696572223A2022746F705F73756273637269626572735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='traffic_general_stat' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_general_stat_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-26T10:54:07.120Z', 'traffic_general_stat', '2023-12-26T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-26T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F67656E6572616C5F73746174222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D32365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F67656E6572616C5F737461742F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D32365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C767379735F6964222C20226D657472696373223A202273657373696F6E732C6163746976655F73657373696F6E732C636C6F7365645F73657373696F6E732C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C6173796D6D65747269635F6332735F666C6F77732C6173796D6D65747269635F7332635F666C6F77732C7463705F73796E5F706B74732C7463705F61636B5F706B74732C7463705F66696E5F706B74732C7463705F7273745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20353235322C20226964656E746966696572223A2022747261666669635F67656E6572616C5F737461745F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D32365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='traffic_shaping_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_shaping_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'traffic_shaping_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F73686170696E675F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F73686170696E675F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C70726F66696C655F69642C7072696F726974792C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C696E5F64726F705F706B74732C6F75745F64726F705F706B74732C696E5F6D61785F6C6174656E63795F75732C6F75745F6D61785F6C6174656E63795F75732C696E5F71756575655F6C656E2C6F75745F71756575655F6C656E222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20343335332C20226964656E746966696572223A2022747261666669635F73686170696E675F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='traffic_top_destination_ip_metrics_log' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_top_destination_ip_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'traffic_top_destination_ip_metrics_log', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F67222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A202261747461636B5F747970652C64657374696E6174696F6E5F69702C6269745F726174652C7061636B65745F726174652C706172746974696F6E5F6E756D2C73657373696F6E5F726174652C767379735F6964222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323139332C20226964656E746966696572223A2022747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/push_segments.sh b/tsg-olap/parcels/roles/initialization/druid/files/cluster/push_segments.sh deleted file mode 100644 index 3a5410e..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/cluster/push_segments.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -source /etc/profile - -TIME_DIR="30000101T000000.000Z_30000102T000000.000Z" - -hadoop fs -test -e /druid/segments -if [ $? -ne 0 ] ;then -hadoop fs -mkdir -p /druid/segments -fi - -for dir in `ls ./segments` -do - hadoop fs -test -e /druid/segments/$dir/$TIME_DIR - if [ $? -eq 0 ] ;then - for file in `ls ./segments/$dir/$TIME_DIR` - do - hadoop fs -test -e /druid/segments/$dir/$TIME_DIR/$file - if [ $? -ne 0 ] ;then - hadoop fs -put ./segments/$dir/$TIME_DIR/$file /druid/segments/$dir/$TIME_DIR/ - fi - done - else - hadoop fs -put ./segments/$dir /druid/segments/ - fi - -done - - diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_data_rule.json b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_data_rule.json deleted file mode 100644 index ad1c808..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_data_rule.json +++ /dev/null @@ -1,13 +0,0 @@ -[ - { - "type" : "loadByPeriod", - "period" : "P365D", - "includeFuture" : true, - "tieredReplicants": { - "_default_tier" : 1 - } - }, - { - "type" : "dropForever" - } -] diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_task.txt b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_task.txt deleted file mode 100644 index f3e8373..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_task.txt +++ /dev/null @@ -1,19 +0,0 @@ -application_protocol_stat -proxy_rule_hits -security_rule_hits -service_chaining_rule_hits -service_function_status -statistics_object -statistics_rule_hits -statistics_rule -sys_storage_log -top_client_ips -top_external_ips -top_internal_ips -top_server_domains -top_server_fqdns -top_server_ips -top_subscribers -traffic_general_stat -traffic_shaping_rule_hits -traffic_top_destination_ip_metrics_log diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_data_rule.json b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_data_rule.json deleted file mode 100644 index a44486e..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_data_rule.json +++ /dev/null @@ -1,13 +0,0 @@ -[ - { - "type" : "loadByPeriod", - "period" : "PT1H", - "includeFuture" : true, - "tieredReplicants": { - "_default_tier" : 1 - } - }, - { - "type" : "dropForever" - } -] diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_task.txt b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_task.txt deleted file mode 100644 index e69de29..0000000 diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/post_rule.sh b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/post_rule.sh deleted file mode 100644 index 0889f1d..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/post_rule.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -base_dir=`dirname $0` -echo $base_dir - -#router节点IP -router_ip=$1 - -common_task_name=`cat $base_dir/common_task.txt` -common_rule_file="$base_dir/common_data_rule.json" - -hot_task_name=`cat $base_dir/hot_task.txt` -hot_rule_file="$base_dir/hot_data_rule.json" - -for var in ${common_task_name[@]}; -do -curl -X 'POST' -H 'Content-Type:application/json' -d @$common_rule_file http://${router_ip}:8089/druid/coordinator/v1/rules/${var} -echo "'${var}' 任务启动成功" -sleep 2 -done - -for var in ${hot_task_name[@]}; -do -curl -X 'POST' -H 'Content-Type:application/json' -d @$hot_rule_file http://${router_ip}:8089/druid/coordinator/v1/rules/${var} -echo "'${var}' 任务启动成功" -sleep 2 -done - diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/supervisor-manger b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/supervisor-manger deleted file mode 100644 index 799da22..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/supervisor-manger +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -base_dir=`dirname $0` -base_dir="$(cd "$base_dir" && pwd)" -#echo $base_dir - -router_ip='127.0.0.1' - -common_task_name=`cat $base_dir/common_task.txt` -common_rule_file="$base_dir/common_data_rule.json" - -hot_task_name=`cat $base_dir/hot_task.txt` -hot_rule_file="$base_dir/hot_data_rule.json" - -case $1 in - - resetAll) - for var in ${common_task_name[@]}; - do - curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset - sleep 2 - done - - for var in ${hot_task_name[@]}; - do - curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset - sleep 2 - done - ;; - - terminateAll) - curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/terminateAll - ;; - - shutdownAllTasks) - for var in ${common_task_name[@]}; - do - curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks - sleep 2 - done - - for var in ${hot_task_name[@]}; - do - curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks - sleep 2 - done - ;; - - reset) - if [ $# -le 1 ]; then - echo "Usage: supervisor-manger reset " - exit 1 - fi - curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/reset - ;; - - terminate) - if [ $# -le 1 ]; then - echo "Usage: supervisor-manger terminate " - exit 1 - fi - curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/terminate - ;; - - shutdownTasks) - if [ $# -le 1 ]; then - echo "Usage: supervisor-manger shutdownTasks " - exit 1 - fi - curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/$2/shutdownAllTasks - ;; - *) - echo "Usage: supervisor-manger {resetAll|terminateAll|shutdownAllTasks}" - echo "Usage: supervisor-manger {reset|terminate|shutdownTasks} " - ;; -esac diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments.zip deleted file mode 100644 index cacbbe1..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/application_protocol_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/application_protocol_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip deleted file mode 100644 index af00337..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/application_protocol_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/monitor_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/monitor_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip deleted file mode 100644 index 6989ff6..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/monitor_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/object_statistics/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/object_statistics/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip deleted file mode 100644 index 8f6fe84..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/object_statistics/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/proxy_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/proxy_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index ac74b6a..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/proxy_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/security_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/security_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip deleted file mode 100644 index 6989ff6..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/security_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_chaining_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_chaining_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index e5ae0ef..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_chaining_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_function_status/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_function_status/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 5062936..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_function_status/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 2424df4..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 913226a..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 22590a9..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_client_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_client_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 46e2772..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_client_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_external_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_external_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 24cefaa..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_external_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_internal_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_internal_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 79e9fb5..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_internal_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_domains/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_domains/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index ae93e8d..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_domains/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_fqdns/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_fqdns/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 8799887..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_fqdns/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 0926f1f..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_subscribers/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_subscribers/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index e27ddcd..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_subscribers/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_general_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-26T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_general_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-26T10_54_07.120Z/0_index.zip deleted file mode 100644 index 66c55ae..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_general_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-26T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_shaping_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_shaping_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 84c033d..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_shaping_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_top_destination_ip_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_top_destination_ip_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip deleted file mode 100644 index 6305798..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_top_destination_ip_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/mysql b/tsg-olap/parcels/roles/initialization/druid/files/mysql deleted file mode 100644 index eb26146..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/mysql and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/rule/common_data_rule.json b/tsg-olap/parcels/roles/initialization/druid/files/rule/common_data_rule.json deleted file mode 100644 index ad1c808..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/rule/common_data_rule.json +++ /dev/null @@ -1,13 +0,0 @@ -[ - { - "type" : "loadByPeriod", - "period" : "P365D", - "includeFuture" : true, - "tieredReplicants": { - "_default_tier" : 1 - } - }, - { - "type" : "dropForever" - } -] diff --git a/tsg-olap/parcels/roles/initialization/druid/files/rule/common_task.txt b/tsg-olap/parcels/roles/initialization/druid/files/rule/common_task.txt deleted file mode 100644 index 6f6e7f1..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/rule/common_task.txt +++ /dev/null @@ -1,19 +0,0 @@ -traffic_general_stat.json -application_protocol_stat.json -traffic_shaping_rule_hits.json -traffic_top_destination_ip_metrics_log.json -service_chaining_rule_hits.json -security_rule_hits.json -proxy_rule_hits.json -service_function_status.json -sys_storage_log.json -top_client_ips.json -top_subscribers.json -top_external_ips.json -top_server_ips.json -top_internal_ips.json -top_server_fqdns.json -top_server_domains.json -statistics_rule.json -statistics_rule_hits.json -object_statistics.json diff --git a/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_data_rule.json b/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_data_rule.json deleted file mode 100644 index a44486e..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_data_rule.json +++ /dev/null @@ -1,13 +0,0 @@ -[ - { - "type" : "loadByPeriod", - "period" : "PT1H", - "includeFuture" : true, - "tieredReplicants": { - "_default_tier" : 1 - } - }, - { - "type" : "dropForever" - } -] diff --git a/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_task.txt b/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_task.txt deleted file mode 100644 index 9bfa834..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_task.txt +++ /dev/null @@ -1,2 +0,0 @@ -urls_proxy_hot -urls_security_hot diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/druid_segments-tsg.sql b/tsg-olap/parcels/roles/initialization/druid/files/standalone/druid_segments-tsg.sql deleted file mode 100644 index 69118f4..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/files/standalone/druid_segments-tsg.sql +++ /dev/null @@ -1,39 +0,0 @@ -delete FROM druid.druid_segments where dataSource='application_protocol_stat' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('application_protocol_stat_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-19T10:54:07.120Z', 'application_protocol_stat', '2023-12-19T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-19T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226170706C69636174696F6E5F70726F746F636F6C5F73746174222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31395431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F6170706C69636174696F6E5F70726F746F636F6C5F737461742F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D31395431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C70726F746F636F6C5F737461636B5F69642C6170705F6E616D652C767379735F6964222C20226D657472696373223A202273657373696F6E732C6332735F706B74732C7332635F706B74732C6332735F62797465732C7332635F62797465732C6332735F667261676D656E74732C7332635F667261676D656E74732C6332735F7463705F6C6F73745F62797465732C7332635F7463705F6C6F73745F62797465732C6332735F7463705F6F6F6F726465725F706B74732C7332635F7463705F6F6F6F726465725F706B74732C6332735F7463705F72657472616E736D69747465645F706B74732C7332635F7463705F72657472616E736D69747465645F706B74732C6332735F7463705F72657472616E736D69747465645F62797465732C7332635F7463705F72657472616E736D69747465645F62797465732C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20373739312C20226964656E746966696572223A20226170706C69636174696F6E5F70726F746F636F6C5F737461745F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31395431303A35343A30372E3132305A227D); -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('monitor_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-12T10:54:07.120Z', 'monitor_rule_hits', '2023-12-12T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-12T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226D6F6E69746F725F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31325431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F6D6F6E69746F725F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D31325431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C616374696F6E222C20226D657472696373223A20226869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333337312C20226964656E746966696572223A20226D6F6E69746F725F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31325431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='object_statistics' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('object_statistics_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-19T10:54:07.120Z', 'object_statistics', '2023-12-19T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-19T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226F626A6563745F73746174697374696373222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31395431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F6F626A6563745F737461746973746963732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D31395431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C74656D706C6174655F69642C63686172745F69642C76657273696F6E2C6F626A6563745F747970652C6F626A6563745F69642C6974656D5F6964222C20226D657472696373223A2022696E5F62797465732C6F75745F62797465732C62797465732C6E65775F696E5F73657373696F6E732C6E65775F6F75745F73657373696F6E732C73657373696F6E73222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20343631372C20226964656E746966696572223A20226F626A6563745F737461746973746963735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31395431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='proxy_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('proxy_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'proxy_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A202270726F78795F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F70726F78795F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C7375625F616374696F6E2C70696E6E696E675F7374617475732C72756C655F69642C616374696F6E2C6869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333736352C20226964656E746966696572223A202270726F78795F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('security_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-12T10:54:07.120Z', 'security_rule_hits', '2023-12-12T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-12T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A202273656375726974795F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31325431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F73656375726974795F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D31325431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C616374696F6E222C20226D657472696373223A20226869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333337312C20226964656E746966696572223A202273656375726974795F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31325431303A35343A30372E3132305A227D); - -delete FROM druid.druid_segments where dataSource='service_chaining_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('service_chaining_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'service_chaining_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022736572766963655F636861696E696E675F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F736572766963655F636861696E696E675F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C7366665F70726F66696C655F69642C73665F70726F66696C655F69642C73656E745F706B74732C73656E745F62797465732C726563765F706B74732C726563765F6279746573222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333132322C20226964656E746966696572223A2022736572766963655F636861696E696E675F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='service_function_status' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('service_function_status_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'service_function_status', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022736572766963655F66756E6374696F6E5F737461747573222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F736572766963655F66756E6374696F6E5F7374617475732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C73665F70726F66696C655F69642C73665F7374617475732C73665F6C6174656E63795F7573222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323333312C20226964656E746966696572223A2022736572766963655F66756E6374696F6E5F7374617475735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='statistics_rule' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('statistics_rule_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'statistics_rule', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022737461746973746963735F72756C65222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F737461746973746963735F72756C652F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C74656D706C6174655F69642C63686172745F69642C76657273696F6E2C636C69656E745F69705F6F626A6563742C7365727665725F69705F6F626A6563742C6671646E5F63617465676F72792C636C69656E745F69702C7365727665725F69702C7365727665725F6671646E2C7365727665725F646F6D61696E2C6170706C69636174696F6E2C696E5F62797465732C6F75745F62797465732C62797465732C6E65775F6332735F666C6F77732C6E65775F7332635F666C6F77732C6E65775F696E5F73657373696F6E732C6E65775F6F75745F73657373696F6E732C73657373696F6E732C6E65775F756E65737461626C69736865645F73657373696F6E732C73796E5F706B7473222C20226D657472696373223A202273657373696F6E5F6964656E7469666965725F736B657463682C636C69656E745F69705F736B657463682C7365727665725F69705F736B657463682C696E5F6C6174656E63795F6D735F736B657463682C6F75745F6C6174656E63795F6D735F736B657463682C6C6174656E63795F6D735F736B65746368222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20393433362C20226964656E746966696572223A2022737461746973746963735F72756C655F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='statistics_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('statistics_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'statistics_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022737461746973746963735F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F737461746973746963735F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C6869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323838302C20226964656E746966696572223A2022737461746973746963735F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='sys_storage_log' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('sys_storage_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'sys_storage_log', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20227379735F73746F726167655F6C6F67222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F7379735F73746F726167655F6C6F672F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226C6F675F747970652C646174615F63656E7465722C6167677265676174655F73697A652C6C6173745F73746F726167652C6D61785F73697A652C757365645F73697A65222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20313938322C20226964656E746966696572223A20227379735F73746F726167655F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_client_ips' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_client_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_client_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F636C69656E745F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F636C69656E745F6970732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022636C69656E745F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333335362C20226964656E746966696572223A2022746F705F636C69656E745F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_external_ips' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_external_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_external_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F65787465726E616C5F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F65787465726E616C5F6970732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A202265787465726E616C5F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336322C20226964656E746966696572223A2022746F705F65787465726E616C5F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_internal_ips' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_internal_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_internal_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F696E7465726E616C5F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F696E7465726E616C5F6970732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022696E7465726E616C5F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336322C20226964656E746966696572223A2022746F705F696E7465726E616C5F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_server_domains' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_domains_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_domains', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F646F6D61696E73222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F7365727665725F646F6D61696E732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022646F6D61696E2C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333334372C20226964656E746966696572223A2022746F705F7365727665725F646F6D61696E735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_server_fqdns' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_fqdns_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_fqdns', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F6671646E73222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F7365727665725F6671646E732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226671646E2C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333334312C20226964656E746966696572223A2022746F705F7365727665725F6671646E735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_server_ips' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F7365727665725F6970732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20227365727665725F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333335362C20226964656E746966696572223A2022746F705F7365727665725F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='top_subscribers' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_subscribers_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_subscribers', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7375627363726962657273222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F73756273637269626572732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022737562736372696265725F69642C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336382C20226964656E746966696572223A2022746F705F73756273637269626572735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='traffic_general_stat' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_general_stat_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-26T10:54:07.120Z', 'traffic_general_stat', '2023-12-26T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-26T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F67656E6572616C5F73746174222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D32365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F747261666669635F67656E6572616C5F737461742F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D32365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C767379735F6964222C20226D657472696373223A202273657373696F6E732C6163746976655F73657373696F6E732C636C6F7365645F73657373696F6E732C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C6173796D6D65747269635F6332735F666C6F77732C6173796D6D65747269635F7332635F666C6F77732C7463705F73796E5F706B74732C7463705F61636B5F706B74732C7463705F66696E5F706B74732C7463705F7273745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20353235322C20226964656E746966696572223A2022747261666669635F67656E6572616C5F737461745F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D32365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='traffic_shaping_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_shaping_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'traffic_shaping_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F73686170696E675F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F747261666669635F73686170696E675F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C70726F66696C655F69642C7072696F726974792C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C696E5F64726F705F706B74732C6F75745F64726F705F706B74732C696E5F6D61785F6C6174656E63795F75732C6F75745F6D61785F6C6174656E63795F75732C696E5F71756575655F6C656E2C6F75745F71756575655F6C656E222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20343335332C20226964656E746966696572223A2022747261666669635F73686170696E675F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); -delete FROM druid.druid_segments where dataSource='traffic_top_destination_ip_metrics_log' and `start` = '3000-01-02T00:00:00.000Z'; -REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_top_destination_ip_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'traffic_top_destination_ip_metrics_log', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F67222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F672F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A202261747461636B5F747970652C64657374696E6174696F6E5F69702C6269745F726174652C7061636B65745F726174652C706172746974696F6E5F6E756D2C73657373696F6E5F726174652C767379735F6964222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323139332C20226964656E746966696572223A2022747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments.zip deleted file mode 100644 index 7065ed6..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/application_protocol_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-12-19T10_54_07.120Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/application_protocol_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-12-19T10_54_07.120Z/0/index.zip deleted file mode 100644 index af00337..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/application_protocol_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-12-19T10_54_07.120Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/object_statistics/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-24T02_43_00.245Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/object_statistics/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-24T02_43_00.245Z/0/index.zip deleted file mode 100644 index 315ce74..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/object_statistics/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-24T02_43_00.245Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/proxy_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-10-18T10_44_50.148Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/proxy_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-10-18T10_44_50.148Z/0/index.zip deleted file mode 100644 index fc74b52..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/proxy_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-10-18T10_44_50.148Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/security_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-21T10_23_31.391Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/security_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-21T10_23_31.391Z/0/index.zip deleted file mode 100644 index db20bfb..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/security_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-21T10_23_31.391Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_chaining_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-17T06_26_28.371Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_chaining_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-17T06_26_28.371Z/0/index.zip deleted file mode 100644 index 1218d51..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_chaining_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-17T06_26_28.371Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_function_status/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-02-10T03_07_09.579Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_function_status/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-02-10T03_07_09.579Z/0/index.zip deleted file mode 100644 index 4d98e20..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_function_status/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-02-10T03_07_09.579Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-07-28T07_05_53.280Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-07-28T07_05_53.280Z/0/index.zip deleted file mode 100644 index 7a46b17..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-07-28T07_05_53.280Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-04T01_58_01.118Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-04T01_58_01.118Z/0/index.zip deleted file mode 100644 index e45f8a9..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-04T01_58_01.118Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/sys_storage_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-02-04T03_40_23.597Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/sys_storage_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-02-04T03_40_23.597Z/0/index.zip deleted file mode 100644 index 7958e35..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/sys_storage_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-02-04T03_40_23.597Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-23T07_38_34.276Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-23T07_38_34.276Z/0/index.zip deleted file mode 100644 index 6e66708..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-23T07_38_34.276Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T09_35_24.409Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T09_35_24.409Z/0/index.zip deleted file mode 100644 index 0d6cf04..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T09_35_24.409Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-05T02_14_42.249Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-05T02_14_42.249Z/0/index.zip deleted file mode 100644 index a5236eb..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-05T02_14_42.249Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_external_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_37.283Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_external_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_37.283Z/0/index.zip deleted file mode 100644 index 9bd0145..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_external_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_37.283Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_internal_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_30.600Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_internal_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_30.600Z/0/index.zip deleted file mode 100644 index fc955b0..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_internal_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_30.600Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_domains/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T06_21_09.765Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_domains/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T06_21_09.765Z/0/index.zip deleted file mode 100644 index 5210fee..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_domains/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T06_21_09.765Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_fqdns/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T09_15_58.356Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_fqdns/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T09_15_58.356Z/0/index.zip deleted file mode 100644 index 4c9c033..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_fqdns/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T09_15_58.356Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T07_40_42.170Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T07_40_42.170Z/0/index.zip deleted file mode 100644 index 7fac0da..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T07_40_42.170Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_subscribers/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_40_14.760Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_subscribers/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_40_14.760Z/0/index.zip deleted file mode 100644 index 2b6a611..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_subscribers/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_40_14.760Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-18T02_05_57.690Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-18T02_05_57.690Z/0/index.zip deleted file mode 100644 index 90e2d7f..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-18T02_05_57.690Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_19_29.661Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_19_29.661Z/0/index.zip deleted file mode 100644 index 4e201d2..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_19_29.661Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_43_09.655Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_43_09.655Z/0/index.zip deleted file mode 100644 index e21f94b..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_43_09.655Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_50_55.510Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_50_55.510Z/0/index.zip deleted file mode 100644 index 6964bd4..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_50_55.510Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_52_58.683Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_52_58.683Z/0/index.zip deleted file mode 100644 index 91ff68a..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_52_58.683Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_53_44.028Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_53_44.028Z/0/index.zip deleted file mode 100644 index aad2f86..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_53_44.028Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_06_43.540Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_06_43.540Z/0/index.zip deleted file mode 100644 index b4c4600..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_06_43.540Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_26_53.933Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_26_53.933Z/0/index.zip deleted file mode 100644 index 51dd724..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_26_53.933Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_50_48.464Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_50_48.464Z/0/index.zip deleted file mode 100644 index 9fbb526..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_50_48.464Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_shaping_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-03-30T07_05_43.972Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_shaping_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-03-30T07_05_43.972Z/0/index.zip deleted file mode 100644 index df83b3e..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_shaping_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-03-30T07_05_43.972Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_10_50.930Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_10_50.930Z/0/index.zip deleted file mode 100644 index 48e44f6..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_10_50.930Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_15_36.184Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_15_36.184Z/0/index.zip deleted file mode 100644 index 7467dc8..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_15_36.184Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2022-09-06T06_57_25.946Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2022-09-06T06_57_25.946Z/0/index.zip deleted file mode 100644 index 51dbbf7..0000000 Binary files a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2022-09-06T06_57_25.946Z/0/index.zip and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml deleted file mode 100644 index 899461b..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml +++ /dev/null @@ -1,25 +0,0 @@ -- name: Check if backup directory exists - stat: - path: "{{ backup_path }}" - register: exist_status - -- name: Backup directories - block: - - name: Creat backup directory - file: - state: directory - path: "{{ backup_path }}" - - - name: Copying directories to backup - copy: - src: "{{ deploy_dir }}/{{ container_name }}/{{ item }}" - dest: "{{ backup_path }}" - remote_src: true - loop: "{{ backup_items }}" - - rescue: - - name: Remove backup directory on failure - file: - path: "{{ backup_path }}" - state: absent - when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml deleted file mode 100644 index 295e895..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml +++ /dev/null @@ -1,23 +0,0 @@ -- name: Get druid running tasks - shell: curl -s http://{{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/indexer/v1/supervisor?state=true | jq -r '.[] | select(.state == "RUNNING") | .id' - register: run_task_list - run_once: true - -- name: Get init task template - find: - paths: "templates/tasks/" - file_type: file - patterns: "*.json" - register: find_result - delegate_to: 127.0.0.1 - run_once: true - -- name: Set init task list - set_fact: - init_task_list: "{{ find_result.files | map(attribute='path') | map('basename') | map('regex_replace', '^index_kafka_(.*)\\.json$', '\\1') | list }}" - -- name: Verify that the number of running tasks is correct - fail: - msg: "The number of running druid tasks is less than the normal number!" - when: run_task_list.stdout_lines | intersect(init_task_list) |length != (init_task_list|length) - run_once: true diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml deleted file mode 100644 index aaff071..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml +++ /dev/null @@ -1,115 +0,0 @@ -- name: Setting init directory - set_fact: - init_path: '{{ deploy_dir }}/druid/init/' - topology_path: '{{ deploy_dir }}/druid/topology/' - -- name: Remove files and directories - file: - path: "{{ init_path }}" - state: absent - ignore_errors: true - loop: - - "{{ init_path }}" - - "{{ topology_path }}" - -- name: Creatting init directory - file: - path: '{{ item }}' - state: directory - loop: - - "{{ init_path }}" - - "{{ topology_path }}" - - "{{ topology_path }}/tasks" - -- name: Setting hdfs_ip variable - set_fact: - hdfs_ip: "{{groups.hdfs[0]}}" - -- name: Unzipping segments.zip - unarchive: - src: 'files/cluster/segments.zip' - dest: '{{ deploy_dir }}/' - force: true - run_once: true - delegate_to: '{{ hdfs_ip }}' - -- name: Copying push segments to hdfs installation path - copy: - src: 'files/cluster/push_segments.sh' - dest: '{{ deploy_dir }}/push_segments.sh' - force: true - mode: 0755 - run_once: true - delegate_to: '{{ hdfs_ip }}' - -- name: Creating druid directory and putting segments to hdfs - shell: source /etc/profile && hadoop fs -mkdir -p /druid/segments - register: nums_out - run_once: true - delegate_to: '{{ hdfs_ip }}' - ignore_errors: yes - -- name: Putting segments to hdfs - shell: source /etc/profile && cd {{ deploy_dir }}/ && sh push_segments.sh - register: nums_out - run_once: true - delegate_to: '{{ hdfs_ip }}' - -- name: Copying mysql Client to /usr/bin/ - copy: - src: 'files/mysql' - dest: '/usr/bin/' - mode: 0755 - force: true - -- name: Copying init files to {{ init_path }} - copy: - src: 'files/cluster/' - dest: '{{ deploy_dir }}/druid/init' - force: true - -- name: Copying tasks template to {{ topology_path }} - template: - src: "{{ item }}" - dest: "{{ deploy_dir }}/druid/topology/tasks/{{ item | basename | regex_replace('.j2$', '') }}" - mode: "0644" - force: true - with_fileglob: "templates/tasks/*" - -- name: Inserting segments to mariadb - shell: 'mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} druid < {{ init_path }}/druid_segments-tsg.sql' - run_once: true - -- name: Copying script files - template: - src: '{{ item }}.j2' - dest: '{{ topology_path }}/{{ item }}' - mode: 0755 - with_items: - - 'druid_monitor.sh' - - 'supervisor-manager.sh' - -- name: Copying monitor template - template: - src: 'run_druid_monitor.j2' - dest: '/etc/cron.d/run_druid_monitor' - mode: 0644 - -- name: Submitting druid tasks - shell: cd {{ topology_path }} && sh supervisor-manager.sh startall - run_once: true - -- name: Deleting {{ deploy_dir }}/push_segments.sh - file: - path: "{{ deploy_dir }}/push_segments.sh" - state: absent - run_once: true - delegate_to: '{{ hdfs_ip }}' - -- name: Deleting {{ deploy_dir }}/segments.zip - file: - path: "{{ deploy_dir }}/segments.zip" - state: absent - run_once: true - delegate_to: '{{ hdfs_ip }}' - diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml deleted file mode 100644 index 9c10b77..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml +++ /dev/null @@ -1,65 +0,0 @@ -- name: Setting init directory - set_fact: - init_path: '{{ deploy_dir }}/druid/init/' - topology_path: '{{ deploy_dir }}/druid/topology/' - -- name: Creatting init directory - file: - path: '{{ item }}' - state: directory - loop: - - '{{ init_path }}' - - '{{ topology_path }}' - - '{{ topology_path }}/tasks' - - '{{ deploy_dir }}/druid/var/druid' - -- name: Unzipping segments.zip - unarchive: - src: 'files/standalone/segments.zip' - dest: '{{ deploy_dir }}/druid/var/druid/' - force: true - -- name: Copying mysql Client to /usr/bin/ - copy: - src: 'files/mysql' - dest: '/usr/bin/' - mode: 0755 - force: true - -- name: Copying init files to {{ init_path }} - copy: - src: 'files/standalone' - dest: '{{ init_path }}/' - force: true - -- name: Copying tasks template to {{ topology_path }} - template: - src: "{{ item }}" - dest: "{{ deploy_dir }}/druid/topology/tasks/{{ item | basename | regex_replace('.j2$', '') }}" - mode: "0644" - force: true - with_fileglob: "templates/tasks/*" - - -- name: Inserting segments to mariadb - shell: 'mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} druid < {{ init_path }}/standalone/druid_segments-tsg.sql' - run_once: true - -- name: Copying script files - template: - src: '{{ item }}.j2' - dest: '{{ topology_path }}/{{ item }}' - mode: 0755 - with_items: - - "druid_monitor.sh" - - "supervisor-manager.sh" - -- name: Copying monitor template - template: - src: 'run_druid_monitor.j2' - dest: '/etc/cron.d/run_druid_monitor' - mode: 0644 - -- name: Submitting druid tasks - shell: cd {{ topology_path }} && sh supervisor-manager.sh startall - run_once: true diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml deleted file mode 100644 index 35bf1ff..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -- block: - - include: uninstall.yml - - include: "{{ playbook_name }}" - vars: - playbook_name: "{{ 'init-cluster.yml' if groups.druid | length > 1 else 'init-standalone.yml' }}" - - include: check.yml - when: inventory_hostname in groups['druid'][:2] and operation == 'install' - -- block: - - include: backup.yml - - include: uninstall.yml - - include: "{{ playbook_name }}" - vars: - playbook_name: "{{ 'init-cluster.yml' if groups.druid | length > 1 else 'init-standalone.yml' }}" - - include: check.yml - when: inventory_hostname in groups['druid'][:2] and (operation) == "upgrade" - -- block: - - include: uninstall.yml - when: inventory_hostname in groups['druid'][:2] and (operation) == "uninstall" - diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml deleted file mode 100644 index ec46d7c..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml +++ /dev/null @@ -1,30 +0,0 @@ -- name: Check {{ deploy_dir }}/druid/topology - stat: - path: "{{ deploy_dir }}/druid/topology" - register: directory_stat - -- block: - - name: Copying supervisor-manager.sh template - template: - src: 'supervisor-manager.sh.j2' - dest: '{{ deploy_dir }}/druid/topology/supervisor-manager.sh' - mode: 0755 - - - name: Stop all tasks - shell: cd {{ deploy_dir }}/druid/topology && sh supervisor-manager.sh terminateall - run_once: true - - - name: Get tasks status - shell: curl -s http://{{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/indexer/v1/supervisor?state=true | jq 'map(select(.state = "RUNNING" ))' | jq 'length' - register: tasks_count - - - name: Check tasks status - debug: - msg: "{{ tasks_count.stdout }} tasks are not stopped" - failed_when: tasks_count.stdout > '0' - - - name: Delete {{ deploy_dir }}/druid/topology - file: - path: "{{ deploy_dir }}/druid/topology" - state: absent - when: directory_stat.stat.exists diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 deleted file mode 100644 index e0c0444..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -druid_ip={{ vrrp_instance.default.virtual_ipaddress }} -druid_port=8089 -druid_monitor_prom_file={{ deploy_dir }}/node-exporter/prom/druid_monitor.prom - -task_name=`cat tasklist` - -#获取supervisor状态信息 -function getSupervisorStatus(){ -druid_supervisor_healthy_num=`curl -G -d 'state=true' -s $druid_ip:$druid_port/druid/indexer/v1/supervisor | jq '[ .[] | .state] | length'` -echo druid_supervisor_healthy_num $druid_supervisor_healthy_num >> $druid_monitor_prom_file - -druid_supervisor_unhealthy_num=`curl -G -d 'state=true' -s $druid_ip:$druid_port/druid/indexer/v1/supervisor | jq '[ .[] | select(.state != "RUNNING")] | length'` -echo druid_supervisor_unhealthy_num $druid_supervisor_unhealthy_num >> $druid_monitor_prom_file -} - -#获取Druid 任务状态信息 -function getTaskStatus(){ -druid_index_running_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/runningTasks | jq '. | length'` -echo druid_index_running_task_num $druid_index_running_task_num > $druid_monitor_prom_file - -druid_index_waiting_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/waitingTasks | jq '. | length'` -echo druid_index_waiting_task_num $druid_index_waiting_task_num >> $druid_monitor_prom_file - -druid_index_pending_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/pendingTasks | jq '. | length'` -echo druid_index_pending_task_num $druid_index_pending_task_num >> $druid_monitor_prom_file - -druid_compact_pending_task_num=`curl -G -d 'type=compact' -s $druid_ip:$druid_port/druid/indexer/v1/pendingTasks | jq '. | length'` -echo druid_compact_pending_task_num $druid_compact_pending_task_num >> $druid_monitor_prom_file - -druid_compact_waiting_task_num=`curl -G -d 'type=compact' -s $druid_ip:$druid_port/druid/indexer/v1/waitingTasks | jq '. | length'` -echo druid_compact_waiting_task_num $druid_compact_waiting_task_num >> $druid_monitor_prom_file -} - -#校验正在运行的index数量,与启动时指定的index数量对比。 -#>=指定数值判断为正常 返回:0 -#反之异常 返回:1 -function checkIndexStatus(){ -druid_task_sum=0 -for var in ${task_name[@]}; -do - druid_task_num=`cat tasks/${var} | jq .ioConfig.taskCount` - druid_task_sum=`expr $druid_task_num + $druid_task_sum` -done - -druid_index_task_healthy_flag=0 -if [ $druid_index_running_task_num -lt $druid_task_sum ] -then - druid_index_task_healthy_flag=1 -fi - -echo druid_index_task_healthy_flag $druid_index_task_healthy_flag >> $druid_monitor_prom_file -} - -getTaskStatus -getSupervisorStatus -checkIndexStatus diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 deleted file mode 100644 index bbffdc4..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -base_dir=`dirname $0` -echo $base_dir - -#router节点IP -router_ip={{ groups.druid[0] }} - -common_task_name=`cat $base_dir/common_task.txt` -common_rule_file="$base_dir/common_data_rule.json" - -hot_task_name=`cat $base_dir/hot_task.txt` -hot_rule_file="$base_dir/hot_data_rule.json" - -for var in ${common_task_name[@]}; -do -curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @$common_rule_file http://${router_ip}:8088/druid/coordinator/v1/rules/${var} -echo "'${var}' 任务启动成功" -sleep 2 -done - -for var in ${hot_task_name[@]}; -do -curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @$hot_rule_file http://${router_ip}:8088/druid/coordinator/v1/rules/${var} -echo "'${var}' 任务启动成功" -sleep 2 -done - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 deleted file mode 100644 index 88da982..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -base_dir=`dirname $0` -base_dir="$(cd "$base_dir" && pwd)" - -router_ip='{{ vrrp_instance.default.virtual_ipaddress }}' - -common_task_name=`cat $base_dir/tasklist` -common_rule_file="$base_dir/common_data_rule.json" - -hot_task_name=`cat $base_dir/hot_task.txt` -hot_rule_file="$base_dir/hot_data_rule.json" - -case $1 in - - resetAll) - for var in ${common_task_name[@]}; - do - curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset - sleep 2 - done - - for var in ${hot_task_name[@]}; - do - curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset - sleep 2 - done - ;; - - terminateAll) - curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/terminateAll - ;; - - shutdownAllTasks) - for var in ${common_task_name[@]}; - do - curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks - sleep 2 - done - - for var in ${hot_task_name[@]}; - do - curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks - sleep 2 - done - ;; - - reset) - if [ $# -le 1 ]; then - echo "Usage: supervisor-manger reset " - exit 1 - fi - curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/reset - ;; - - terminate) - if [ $# -le 1 ]; then - echo "Usage: supervisor-manger terminate " - exit 1 - fi - curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/terminate - ;; - - shutdownTasks) - if [ $# -le 1 ]; then - echo "Usage: supervisor-manger shutdownTasks " - exit 1 - fi - curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/$2/shutdownAllTasks - ;; - *) - echo "Usage: supervisor-manger {resetAll|terminateAll|shutdownAllTasks}" - echo "Usage: supervisor-manger {reset|terminate|shutdownTasks} " - ;; -esac diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 deleted file mode 100644 index d5fee98..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 +++ /dev/null @@ -1,3 +0,0 @@ -#Ansible: Check druid up to node_exporter -*/10 * * * * root cd {{ topology_path }} && sh druid_monitor.sh - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/s.bak b/tsg-olap/parcels/roles/initialization/druid/templates/s.bak deleted file mode 100644 index ed547af..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/s.bak +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -source /etc/profile - -DRUID_HOST='{{ vrrp_instance.default.virtual_ipaddress }}' -KAFKA_HOST="{% for host in groups['kafka'] %} -{{ host }}:9094{% if not loop.last %},{% endif %} -{% endfor %}" - -# 参数:1.操作 2.任务文件 -OPERATION=$1 - -TASK_LIST=( - index_kafka_application_protocol_stat.json - index_kafka_monitor_rule_hits.json - index_kafka_object_statistics.json - index_kafka_proxy_rule_hits.json - index_kafka_security_rule_hits.json - index_kafka_service_chaining_rule_hits.json - index_kafka_service_function_status.json - index_kafka_statistics_rule_hits.json - index_kafka_statistics_rule.json - index_kafka_sys_storage_log.json - index_kafka_top_client_ips.json - index_kafka_top_external_ips.json - index_kafka_top_internal_ips.json - index_kafka_top_server_domains.json - index_kafka_top_server_fqdns.json - index_kafka_top_server_ips.json - index_kafka_top_subscribers.json - index_kafka_traffic_general_stat.json - index_kafka_traffic_shaping_rule_hits.json - index_kafka_traffic_top_destination_ip_metrics_log.json -) - -if [ $# -lt 1 ]; then - echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" - echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] " - exit 1 -fi - -function start_task() { - local task_name=$1 - cp ./tasks/${task_name} ./ - - sed -i 's/kafkabootstrap/'$KAFKA_HOST'/' ${task_name} - - curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @${task_name} http://$DRUID_HOST:8089/druid/indexer/v1/supervisor - - if [ $? -eq "0" ]; then - echo "task:'${task_name}' Start-up success! " - rm -rf ./${task_name} - else - echo "task:'${task_name}' Start-up failure! " - exit 1 - fi - -} - -function reset_task() { - local task_name=$1 - task_name="${task_name/index_kafka_/}" - task_name="${task_name/.json/}" - curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/reset" - sleep 2 -} - -function terminate_task() { - local task_name=$1 - task_name="${task_name/index_kafka_/}" - task_name="${task_name/.json/}" - curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/terminate" -} - -function terminate_all() { - curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/terminateAll" -} - -function shutdown_tasks() { - local task_name=$1 - task_name="${task_name/index_kafka_/}" - task_name="${task_name/.json/}" - local task_name=$1 - curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/datasources/${task_name}/shutdownAllTasks" - sleep 2 -} - -case $OPERATION in -start) - start_task "$2" - ;; -startall) - for task in ${TASK_LIST[@]}; do - start_task "$task" - done - ;; -reset) - reset_task "$2" - ;; -resetall) - for task in ${TASK_LIST[@]}; do - reset_task "$task" - done - ;; -terminate) - terminate_task "$2" - ;; -terminateall) - terminate_all - ;; -shutdown) - shutdown_tasks "$2" - ;; -shutdownall) - for task in ${TASK_LIST[@]}; do - shutdown_tasks "$task" - done - ;; - -*) - echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" - echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] " - ;; -esac - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 deleted file mode 100644 index e140a07..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -source /etc/profile - -DRUID_HOST='{{ vrrp_instance.default.virtual_ipaddress }}' -KAFKA_HOST="{% for host in groups['kafka'] %} -{{ host }}:9094{% if not loop.last %},{% endif %} -{% endfor %}" - -# 参数:1.操作 2.任务文件或目录 -OPERATION=$1 -filename=$(basename "$file") -TASK_FILE_LIST=$(find "./tasks" -name "*.json") - -if [ $# -lt 1 ]; then - echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" - echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <./tasks/[task file]>" - exit 1 -fi - -function start_task() { - local task_file=$1 - local task_name=$(basename $task_file) - cp ${task_file} ./ - - sed -i 's/kafkabootstrap/'$KAFKA_HOST'/' ${task_name} - - curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @${task_name} http://$DRUID_HOST:8089/druid/indexer/v1/supervisor - - if [ $? -eq "0" ]; then - echo "task:'${task_name}' Start-up success! " - rm -rf ./${task_name} - else - echo "task:'${task_name}' Start-up failure! " - exit 1 - fi - -} - -function reset_task() { - local task_name=$(basename $1) - task_name="${task_name/index_kafka_/}" - task_name="${task_name/.json/}" - curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/reset" - sleep 2 -} - -function terminate_task() { - local task_name=$(basename $1) - task_name="${task_name/index_kafka_/}" - task_name="${task_name/.json/}" - curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/terminate" -} - -function terminate_all() { - curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/terminateAll" -} - -function shutdown_tasks() { - local task_name=$(basename $1) - task_name="${task_name/index_kafka_/}" - task_name="${task_name/.json/}" - curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/datasources/${task_name}/shutdownAllTasks" - sleep 2 -} - -case $OPERATION in -start) - start_task "$2" - ;; -startall) - for task in ${TASK_FILE_LIST[@]}; do - start_task "$task" - done - ;; -reset) - reset_task "$2" - ;; -resetall) - for task in ${TASK_FILE_LIST[@]}; do - reset_task "$task" - done - ;; -terminate) - terminate_task "$2" - ;; -terminateall) - terminate_all - ;; -shutdown) - shutdown_tasks "$2" - ;; -shutdownall) - for task in ${TASK_FILE_LIST[@]}; do - shutdown_tasks "$task" - done - ;; - -*) - echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" - echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <./tasks/[task file]>" - ;; -esac - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json deleted file mode 100644 index 96f992a..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "application_protocol_stat", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - "device_id", - "device_group", - "data_center", - "protocol_stack_id", - "app_name", - {"name": "vsys_id", "type": "long"} - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - {"name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id"}, - {"name": "data_center", "type": "path", "expr": "$.tags.data_center"}, - {"name": "device_group", "type": "path", "expr": "$.tags.device_group"}, - {"name": "device_id", "type": "path", "expr": "$.tags.device_id"}, - {"name": "protocol_stack_id", "type": "path", "expr": "$.tags.protocol_stack_id"}, - {"name": "app_name", "type": "path", "expr": "$.tags.app_name"}, - {"name": "sessions", "type": "path", "expr": "$.fields.sessions"}, - {"name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes"}, - {"name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes"}, - {"name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts"}, - {"name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts"}, - {"name": "c2s_pkts", "type": "path", "expr": "$.fields.c2s_pkts"}, - {"name": "s2c_pkts", "type": "path", "expr": "$.fields.s2c_pkts"}, - {"name": "c2s_bytes", "type": "path", "expr": "$.fields.c2s_bytes"}, - {"name": "s2c_bytes", "type": "path", "expr": "$.fields.s2c_bytes"}, - {"name": "c2s_fragments", "type": "path", "expr": "$.fields.c2s_fragments"}, - {"name": "s2c_fragments", "type": "path", "expr": "$.fields.s2c_fragments"}, - {"name": "c2s_tcp_lost_bytes", "type": "path", "expr": "$.fields.c2s_tcp_lost_bytes"}, - {"name": "s2c_tcp_lost_bytes", "type": "path", "expr": "$.fields.s2c_tcp_lost_bytes"}, - {"name": "c2s_tcp_ooorder_pkts", "type": "path", "expr": "$.fields.c2s_tcp_ooorder_pkts"}, - {"name": "s2c_tcp_ooorder_pkts", "type": "path", "expr": "$.fields.s2c_tcp_ooorder_pkts"}, - {"name": "c2s_tcp_retransmitted_pkts", "type": "path", "expr": "$.fields.c2s_tcp_retransmitted_pkts"}, - {"name": "s2c_tcp_retransmitted_pkts", "type": "path", "expr": "$.fields.s2c_tcp_retransmitted_pkts"}, - {"name": "c2s_tcp_retransmitted_bytes", "type": "path", "expr": "$.fields.c2s_tcp_retransmitted_bytes"}, - {"name": "s2c_tcp_retransmitted_bytes", "type": "path", "expr": "$.fields.s2c_tcp_retransmitted_bytes"} - ] - } - } - }, - "metricsSpec": [ - {"type": "longSum", "name": "sessions", "fieldName": "sessions"}, - {"type": "longSum", "name": "c2s_pkts", "fieldName": "c2s_pkts"}, - {"type": "longSum", "name": "s2c_pkts", "fieldName": "s2c_pkts"}, - {"type": "longSum", "name": "c2s_bytes", "fieldName": "c2s_bytes"}, - {"type": "longSum", "name": "s2c_bytes", "fieldName": "s2c_bytes"}, - {"type": "longSum", "name": "c2s_fragments", "fieldName": "c2s_fragments"}, - {"type": "longSum", "name": "s2c_fragments", "fieldName": "s2c_fragments"}, - {"type": "longSum", "name": "c2s_tcp_lost_bytes", "fieldName": "c2s_tcp_lost_bytes"}, - {"type": "longSum", "name": "s2c_tcp_lost_bytes", "fieldName": "s2c_tcp_lost_bytes"}, - {"type": "longSum", "name": "c2s_tcp_ooorder_pkts", "fieldName": "c2s_tcp_ooorder_pkts"}, - {"type": "longSum", "name": "s2c_tcp_ooorder_pkts", "fieldName": "s2c_tcp_ooorder_pkts"}, - {"type": "longSum", "name": "c2s_tcp_retransmitted_pkts", "fieldName": "c2s_tcp_retransmitted_pkts"}, - {"type": "longSum", "name": "s2c_tcp_retransmitted_pkts", "fieldName": "s2c_tcp_retransmitted_pkts"}, - {"type": "longSum", "name": "c2s_tcp_retransmitted_bytes", "fieldName": "c2s_tcp_retransmitted_bytes"}, - {"type": "longSum", "name": "s2c_tcp_retransmitted_bytes", "fieldName": "s2c_tcp_retransmitted_bytes"}, - {"type": "longSum", "name": "in_bytes", "fieldName": "in_bytes"}, - {"type": "longSum", "name": "out_bytes", "fieldName": "out_bytes"}, - {"type": "longSum", "name": "in_pkts", "fieldName": "in_pkts"}, - {"type": "longSum", "name": "out_pkts", "fieldName": "out_pkts"} - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT15S"}, - "rollup": true - }, - "transformSpec": { - "transforms": [ - {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "c2s_pkts", "expression": "nvl(c2s_pkts, 0)"}, - {"type": "expression", "name": "s2c_pkts", "expression": "nvl(s2c_pkts, 0)"}, - {"type": "expression", "name": "c2s_bytes", "expression": "nvl(c2s_bytes, 0)"}, - {"type": "expression", "name": "s2c_bytes", "expression": "nvl(s2c_bytes, 0)"}, - {"type": "expression", "name": "c2s_fragments", "expression": "nvl(c2s_fragments, 0)"}, - {"type": "expression", "name": "s2c_fragments", "expression": "nvl(s2c_fragments, 0)"}, - {"type": "expression", "name": "c2s_tcp_lost_bytes", "expression": "nvl(c2s_tcp_lost_bytes, 0)"}, - {"type": "expression", "name": "s2c_tcp_lost_bytes", "expression": "nvl(s2c_tcp_lost_bytes, 0)"}, - {"type": "expression", "name": "c2s_tcp_ooorder_pkts", "expression": "nvl(c2s_tcp_ooorder_pkts, 0)"}, - {"type": "expression", "name": "s2c_tcp_ooorder_pkts", "expression": "nvl(s2c_tcp_ooorder_pkts, 0)"}, - {"type": "expression", "name": "c2s_tcp_retransmitted_pkts", "expression": "nvl(c2s_tcp_retransmitted_pkts, 0)"}, - {"type": "expression", "name": "s2c_tcp_retransmitted_pkts", "expression": "nvl(s2c_tcp_retransmitted_pkts, 0)"}, - {"type": "expression", "name": "c2s_tcp_retransmitted_bytes", "expression": "nvl(c2s_tcp_retransmitted_bytes, 0)"}, - {"type": "expression", "name": "s2c_tcp_retransmitted_bytes", "expression": "nvl(s2c_tcp_retransmitted_bytes, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter": { - "type": "selector", - "dimension": "name", - "value": "application_protocol_stat" - } - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 5000000, - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "NETWORK-TRAFFIC-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json deleted file mode 100644 index 5c43f46..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "monitor_rule_hits", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - {"name": "rule_id","type": "long"}, - {"name": "action","type": "long"} - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, - { "name": "action", "type": "path", "expr": "$.tags.action" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1S"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter": { "type": "selector", "dimension": "name", "value": "monitor_rule_hits" } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "POLICY-RULE-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json deleted file mode 100644 index 84dd11b..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "object_statistics", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - {"name": "template_id","type": "long"}, - {"name": "chart_id","type": "long"}, - {"name": "version","type": "long"}, - "object_type", - {"name": "object_id","type": "long"}, - {"name": "item_id","type": "long"} - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "bytes", "type": "path", "expr": "$.fields.bytes" }, - { "name": "new_in_sessions", "type": "path", "expr": "$.fields.new_in_sessions" }, - { "name": "new_out_sessions", "type": "path", "expr": "$.fields.new_out_sessions" }, - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "template_id", "type": "path", "expr": "$.tags.template_id" }, - { "name": "chart_id", "type": "path", "expr": "$.tags.chart_id" }, - { "name": "version", "type": "path", "expr": "$.tags.version" }, - { "name": "object_type", "type": "path", "expr": "$.tags.object_type" }, - { "name": "object_id", "type": "path", "expr": "$.tags.object_id" }, - { "name": "item_id", "type": "path", "expr": "$.tags.item_id" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" }, - { "type" : "longSum", "name" : "new_in_sessions", "fieldName" : "new_in_sessions" }, - { "type" : "longSum", "name" : "new_out_sessions", "fieldName" : "new_out_sessions" }, - { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT15S"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"}, - {"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"}, - {"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"} - ], - "filter": { "type": "selector", "dimension": "name", "value": "object_statistics" } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "OBJECT-STATISTICS-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json deleted file mode 100644 index ab97fb1..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "proxy_rule_hits", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - "device_id", - "device_group", - "data_center", - { "name": "vsys_id", "type": "long" }, - { "name": "rule_id", "type": "long" }, - { "name": "pinning_status", "type": "string" }, - { "name": "action", "type": "long" }, - { "name": "sub_action", "type": "string" } - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "pinning_status", "type": "path", "expr": "$.tags.pinning_status" }, - { "name": "action", "type": "path", "expr": "$.tags.action" }, - { "name": "sub_action", "type": "path", "expr": "$.tags.sub_action" }, - { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1S"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter": { "type": "selector", "dimension": "name", "value": "proxy_rule_hits" } - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 5000000, - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "POLICY-RULE-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json deleted file mode 100644 index 957923d..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "security_rule_hits", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - {"name": "rule_id","type": "long"}, - {"name": "action","type": "long"} - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, - { "name": "action", "type": "path", "expr": "$.tags.action" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1S"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter": { "type": "selector", "dimension": "name", "value": "security_rule_hits" } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "POLICY-RULE-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json deleted file mode 100644 index 6f3e3bd..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "service_chaining_rule_hits", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - "device_id", - "device_group", - "data_center", - { "name": "vsys_id", "type": "long" }, - { "name": "rule_id", "type": "long" }, - { "name": "sff_profile_id", "type": "long" }, - { "name": "sf_profile_id", "type": "long" } - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, - { "name": "sff_profile_id", "type": "path", "expr": "$.tags.sff_profile_id" }, - { "name": "sf_profile_id", "type": "path", "expr": "$.tags.sf_profile_id" }, - { "name": "sent_pkts", "type": "path", "expr": "$.fields.sent_pkts" }, - { "name": "sent_bytes", "type": "path", "expr": "$.fields.sent_bytes" }, - { "name": "recv_pkts", "type": "path", "expr": "$.fields.recv_pkts" }, - { "name": "recv_bytes", "type": "path", "expr": "$.fields.recv_bytes" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "sent_pkts", "fieldName" : "sent_pkts" }, - { "type" : "longSum", "name" : "sent_bytes", "fieldName" : "sent_bytes" }, - { "type" : "longSum", "name" : "recv_pkts", "fieldName" : "recv_pkts" }, - { "type" : "longSum", "name" : "recv_bytes", "fieldName" : "recv_bytes" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1S"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "sent_pkts", "expression": "nvl(sent_pkts, 0)"}, - {"type": "expression", "name": "sent_bytes", "expression": "nvl(sent_bytes, 0)"}, - {"type": "expression", "name": "recv_pkts", "expression": "nvl(recv_pkts, 0)"}, - {"type": "expression", "name": "recv_bytes", "expression": "nvl(recv_bytes, 0)"} - ], - "filter": { "type": "selector", "dimension": "name", "value": "service_chaining_rule_hits" } - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 5000000, - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "POLICY-RULE-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json deleted file mode 100644 index 61e5b12..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "service_function_status", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - "device_id", - "device_group", - "data_center", - { "name": "vsys_id", "type": "long" }, - { "name": "sf_profile_id", "type": "long" } - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "sf_profile_id", "type": "path", "expr": "$.tags.sf_profile_id" }, - { "name": "sf_status", "type": "path", "expr": "$.fields.sf_status" }, - { "name": "sf_latency_us", "type": "path", "expr": "$.fields.sf_latency_us" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longMax", "name" : "sf_latency_us", "fieldName" : "sf_latency_us" }, - { "type" : "longMax", "name" : "sf_status", "fieldName" : "sf_status" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1S"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "sf_latency_us", "expression": "nvl(sf_latency_us, 0)"}, - {"type": "expression", "name": "sf_status", "expression": "nvl(sf_status, 0)"} - ], - "filter": { "type": "selector", "dimension": "name", "value": "service_function_status" } - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 5000000, - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "POLICY-RULE-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json deleted file mode 100644 index 8c15a37..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json +++ /dev/null @@ -1,367 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "statistics_rule", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - "device_id", - "device_group", - "data_center", - { - "name": "vsys_id", - "type": "long" - }, - { - "name": "rule_id", - "type": "long" - }, - { - "name": "template_id", - "type": "long" - }, - { - "name": "chart_id", - "type": "long" - }, - { - "name": "version", - "type": "long" - }, - "client_ip_object", - "server_ip_object", - "fqdn_category", - "client_ip", - "server_ip", - "server_fqdn", - "server_domain", - "application" - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { - "name": "vsys_id", - "type": "path", - "expr": "$.tags.vsys_id" - }, - { - "name": "data_center", - "type": "path", - "expr": "$.tags.data_center" - }, - { - "name": "device_group", - "type": "path", - "expr": "$.tags.device_group" - }, - { - "name": "device_id", - "type": "path", - "expr": "$.tags.device_id" - }, - { - "name": "rule_id", - "type": "path", - "expr": "$.tags.rule_id" - }, - { - "name": "template_id", - "type": "path", - "expr": "$.tags.template_id" - }, - { - "name": "chart_id", - "type": "path", - "expr": "$.tags.chart_id" - }, - { - "name": "version", - "type": "path", - "expr": "$.tags.version" - }, - { - "name": "client_ip_object", - "type": "path", - "expr": "$.tags.client_ip_object" - }, - { - "name": "server_ip_object", - "type": "path", - "expr": "$.tags.server_ip_object" - }, - { - "name": "fqdn_category", - "type": "path", - "expr": "$.tags.fqdn_category" - }, - { - "name": "client_ip", - "type": "path", - "expr": "$.tags.client_ip" - }, - { - "name": "server_ip", - "type": "path", - "expr": "$.tags.server_ip" - }, - { - "name": "server_fqdn", - "type": "path", - "expr": "$.tags.server_fqdn" - }, - { - "name": "server_domain", - "type": "path", - "expr": "$.tags.server_domain" - }, - { - "name": "application", - "type": "path", - "expr": "$.tags.application" - }, - { - "name": "in_bytes", - "type": "path", - "expr": "$.fields.in_bytes" - }, - { - "name": "out_bytes", - "type": "path", - "expr": "$.fields.out_bytes" - }, - { - "name": "bytes", - "type": "path", - "expr": "$.fields.bytes" - }, - { - "name": "new_c2s_flows", - "type": "path", - "expr": "$.fields.new_c2s_flows" - }, - { - "name": "new_s2c_flows", - "type": "path", - "expr": "$.fields.new_s2c_flows" - }, - { - "name": "new_in_sessions", - "type": "path", - "expr": "$.fields.new_in_sessions" - }, - { - "name": "new_out_sessions", - "type": "path", - "expr": "$.fields.new_out_sessions" - }, - { - "name": "sessions", - "type": "path", - "expr": "$.fields.sessions" - }, - { - "name": "new_unestablished_sessions", - "type": "path", - "expr": "$.fields.new_unestablished_sessions" - }, - { - "name": "syn_pkts", - "type": "path", - "expr": "$.fields.syn_pkts" - }, - { - "name": "session_identifier_sketch", - "type": "path", - "expr": "$.fields.session_identifier_sketch" - }, - { - "name": "client_ip_sketch", - "type": "path", - "expr": "$.fields.client_ip_sketch" - }, - { - "name": "server_ip_sketch", - "type": "path", - "expr": "$.fields.server_ip_sketch" - }, - { - "name": "in_latency_ms_sketch", - "type": "path", - "expr": "$.fields.in_latency_ms_sketch" - }, - { - "name": "out_latency_ms_sketch", - "type": "path", - "expr": "$.fields.out_latency_ms_sketch" - }, - { - "name": "latency_ms_sketch", - "type": "path", - "expr": "$.fields.latency_ms_sketch" - } - ] - } - } - }, - "metricsSpec": [ - { - "type": "HLLDSketchMerge", - "name": "session_identifier_sketch", - "fieldName": "session_identifier_sketch" - }, - { - "type": "HLLDSketchMerge", - "name": "client_ip_sketch", - "fieldName": "client_ip_sketch" - },{ - "type": "HLLDSketchMerge", - "name": "server_ip_sketch", - "fieldName": "server_ip_sketch" - }, - { - "type": "HdrHistogramSketchMerge", - "name": "in_latency_ms_sketch", - "fieldName": "in_latency_ms_sketch", - "lowestDiscernibleValue": "1", - "highestTrackableValue": "2", - "numberOfSignificantValueDigits": "3", - "autoResize": "true" - }, - { - "type": "HdrHistogramSketchMerge", - "name": "out_latency_ms_sketch", - "fieldName": "out_latency_ms_sketch", - "lowestDiscernibleValue": "1", - "highestTrackableValue": "2", - "numberOfSignificantValueDigits": "3", - "autoResize": "true" - }, - { - "type": "HdrHistogramSketchMerge", - "name": "latency_ms_sketch", - "fieldName": "latency_ms_sketch", - "lowestDiscernibleValue": "1", - "highestTrackableValue": "2", - "numberOfSignificantValueDigits": "3", - "autoResize": "true" - }, - { - "type": "longSum", - "name": "in_bytes", - "fieldName": "in_bytes" - }, - { - "type": "longSum", - "name": "out_bytes", - "fieldName": "out_bytes" - }, - { - "type": "longSum", - "name": "bytes", - "fieldName": "bytes" - }, - { - "type": "longSum", - "name": "new_c2s_flows", - "fieldName": "new_c2s_flows" - }, - { - "type": "longSum", - "name": "new_s2c_flows", - "fieldName": "new_s2c_flows" - }, - { - "type": "longSum", - "name": "new_in_sessions", - "fieldName": "new_in_sessions" - }, - { - "type": "longSum", - "name": "new_out_sessions", - "fieldName": "new_out_sessions" - }, - { - "type": "longSum", - "name": "sessions", - "fieldName": "sessions" - }, - { - "type": "longSum", - "name": "new_unestablished_sessions", - "fieldName": "new_unestablished_sessions" - }, - { - "type": "longSum", - "name": "syn_pkts", - "fieldName": "syn_pkts" - } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": { - "type": "period", - "period": "PT15S" - }, - "rollup": true - }, - "transformSpec": { - "transforms": [ - {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, - {"type": "expression", "name": "client_ip_object", "expression": "STRING_TO_ARRAY(client_ip_object,',')"}, - {"type": "expression", "name": "server_ip_object", "expression": "STRING_TO_ARRAY(server_ip_object,',')"}, - {"type": "expression", "name": "fqdn_category", "expression": "STRING_TO_ARRAY(fqdn_category,',')"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"}, - {"type": "expression", "name": "new_c2s_flows", "expression": "nvl(new_c2s_flows, 0)"}, - {"type": "expression", "name": "new_s2c_flows", "expression": "nvl(new_s2c_flows, 0)"}, - {"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"}, - {"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "new_unestablished_sessions", "expression": "nvl(new_unestablished_sessions, 0)"}, - {"type": "expression", "name": "syn_pkts", "expression": "nvl(syn_pkts, 0)"} - ], - "filter": { - "type": "selector", - "dimension": "name", - "value": "statistics_rule" - } - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 5000000, - "maxRowsInMemory": 400000, - "maxBytesInMemory": -1, - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "STATISTICS-RULE-METRIC", - "taskCount": 2, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - }, - "context":{ - "druid.indexer.runner.javaOpts": "{{ druid.index_kafka_statistics_rule['druid.indexer.runner.javaOpts'] }}" - } -} - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json deleted file mode 100644 index f431e0f..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "statistics_rule_hits", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - "device_id", - "device_group", - "data_center", - { - "name": "vsys_id", - "type": "long" - }, - { - "name": "rule_id", - "type": "long" - } - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { - "name": "vsys_id", - "type": "path", - "expr": "$.tags.vsys_id" - }, - { - "name": "data_center", - "type": "path", - "expr": "$.tags.data_center" - }, - { - "name": "device_group", - "type": "path", - "expr": "$.tags.device_group" - }, - { - "name": "device_id", - "type": "path", - "expr": "$.tags.device_id" - }, - { - "name": "rule_id", - "type": "path", - "expr": "$.tags.rule_id" - }, - { - "name": "in_bytes", - "type": "path", - "expr": "$.fields.in_bytes" - }, - { - "name": "out_bytes", - "type": "path", - "expr": "$.fields.out_bytes" - }, - { - "name": "hit_count", - "type": "path", - "expr": "$.fields.hit_count" - }, - { - "name": "in_pkts", - "type": "path", - "expr": "$.fields.in_pkts" - }, - { - "name": "out_pkts", - "type": "path", - "expr": "$.fields.out_pkts" - } - ] - } - } - }, - "metricsSpec": [ - { - "type": "longSum", - "name": "in_bytes", - "fieldName": "in_bytes" - }, - { - "type": "longSum", - "name": "out_bytes", - "fieldName": "out_bytes" - }, - { - "type": "longSum", - "name": "hit_count", - "fieldName": "hit_count" - }, - { - "type": "longSum", - "name": "in_pkts", - "fieldName": "in_pkts" - }, - { - "type": "longSum", - "name": "out_pkts", - "fieldName": "out_pkts" - } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": { - "type": "period", - "period": "PT1S" - }, - "rollup": true - }, - "transformSpec": { - "transforms": [ - {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter": { - "type": "selector", - "dimension": "name", - "value": "statistics_rule_hits" - } - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 5000000, - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "POLICY-RULE-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json deleted file mode 100644 index 2b405c2..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "sys_storage_log", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "time", - "format": "posix" - }, - "dimensionsSpec": { - "dimensions": [ - "log_type", - "data_center", - { "name": "used_size", "type": "long" }, - { "name": "max_size", "type": "long" }, - { "name": "aggregate_size", "type": "long" }, - { "name": "last_storage", "type": "long" } - ] - } - } - }, - "metricsSpec" : [ - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, - "rollup": false - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "SYS-STORAGE", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json deleted file mode 100644 index 4a8749f..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "top_client_ips", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - "client_ip", - "metric" - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "client_ip", "type": "path", "expr": "$.tags.client_ip" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "name", "type": "path", "expr": "$.name" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1M"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter":{ - "type":"or","fields":[ - { - "type":"selector", - "dimension":"name", - "value":"packets_top_client_ip" - }, - { - "type":"selector", - "dimension":"name", - "value":"bytes_top_client_ip" - }, - { - "type":"selector", - "dimension":"name", - "value":"sessions_top_client_ip" - } - ] - } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "TRAFFIC-TOP-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json deleted file mode 100644 index 5ff580e..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "top_external_ips", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - "external_ip", - "metric" - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "external_ip", "type": "path", "expr": "$.tags.external_ip" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "name", "type": "path", "expr": "$.name" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1M"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter":{ - "type":"or","fields":[ - { - "type":"selector", - "dimension":"name", - "value":"sessions_top_external_ip" - }, - { - "type":"selector", - "dimension":"name", - "value":"bytes_top_external_ip" - }, - { - "type":"selector", - "dimension":"name", - "value":"packets_top_external_ip" - } - ] - } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "TRAFFIC-TOP-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json deleted file mode 100644 index 8767165..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "top_internal_ips", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - "internal_ip", - "metric" - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "internal_ip", "type": "path", "expr": "$.tags.internal_ip" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "name", "type": "path", "expr": "$.name" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1M"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter":{ - "type":"or","fields":[ - { - "type":"selector", - "dimension":"name", - "value":"sessions_top_internal_ip" - }, - { - "type":"selector", - "dimension":"name", - "value":"bytes_top_internal_ip" - }, - { - "type":"selector", - "dimension":"name", - "value":"packets_top_internal_ip" - } - ] - } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "TRAFFIC-TOP-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json deleted file mode 100644 index bd5b144..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "top_server_domains", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - "domain", - "metric" - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "domain", "type": "path", "expr": "$.tags.domain" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "name", "type": "path", "expr": "$.name" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1M"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter":{ - "type":"or","fields":[ - { - "type":"selector", - "dimension":"name", - "value":"packets_top_server_domain" - }, - { - "type":"selector", - "dimension":"name", - "value":"bytes_top_server_domain" - }, - { - "type":"selector", - "dimension":"name", - "value":"sessions_top_server_domain" - } - ] - } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "TRAFFIC-TOP-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json deleted file mode 100644 index 4bc42af..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "top_server_fqdns", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - "fqdn", - "metric" - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "fqdn", "type": "path", "expr": "$.tags.fqdn" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "name", "type": "path", "expr": "$.name" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1M"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter":{ - "type":"or","fields":[ - { - "type":"selector", - "dimension":"name", - "value":"sessions_top_server_fqdn" - }, - { - "type":"selector", - "dimension":"name", - "value":"bytes_top_server_fqdn" - }, - { - "type":"selector", - "dimension":"name", - "value":"packets_top_server_fqdn" - } - ] - } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "TRAFFIC-TOP-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json deleted file mode 100644 index 1f05adf..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "top_server_ips", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - "server_ip", - "metric" - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "server_ip", "type": "path", "expr": "$.tags.server_ip" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "name", "type": "path", "expr": "$.name" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1M"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter":{ - "type":"or","fields":[ - { - "type":"selector", - "dimension":"name", - "value":"sessions_top_server_ip" - }, - { - "type":"selector", - "dimension":"name", - "value":"bytes_top_server_ip" - }, - { - "type":"selector", - "dimension":"name", - "value":"packets_top_server_ip" - } - ] - } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "TRAFFIC-TOP-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json deleted file mode 100644 index 8bc5e37..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "top_subscribers", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - "subscriber_id", - "metric" - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "subscriber_id", "type": "path", "expr": "$.tags.subscriber_id" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "name", "type": "path", "expr": "$.name" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1M"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} - ], - "filter":{ - "type":"or","fields":[ - { - "type":"selector", - "dimension":"name", - "value":"sessions_top_subscriber_id" - }, - { - "type":"selector", - "dimension":"name", - "value":"bytes_top_subscriber_id" - }, - { - "type":"selector", - "dimension":"name", - "value":"packets_top_subscriber_id" - } - ] - } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "TRAFFIC-TOP-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - - - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json deleted file mode 100644 index 6013977..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "traffic_general_stat", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - "device_id", - "device_group", - "data_center", - { "name": "vsys_id", "type": "long" } - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, - { "name": "active_sessions", "type": "path", "expr": "$.fields.active_sessions" }, - { "name": "closed_sessions", "type": "path", "expr": "$.fields.closed_sessions" }, - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "asymmetric_c2s_flows", "type": "path", "expr": "$.fields.asymmetric_c2s_flows" }, - { "name": "asymmetric_s2c_flows", "type": "path", "expr": "$.fields.asymmetric_s2c_flows" }, - { "name": "tcp_syn_pkts", "type": "path", "expr": "$.fields.tcp_syn_pkts" }, - { "name": "tcp_ack_pkts", "type": "path", "expr": "$.fields.tcp_ack_pkts" }, - { "name": "tcp_fin_pkts", "type": "path", "expr": "$.fields.tcp_fin_pkts" }, - { "name": "tcp_rst_pkts", "type": "path", "expr": "$.fields.tcp_rst_pkts" } - ] - } - } - }, - "metricsSpec":[ - { "type":"longSum", "name":"sessions", "fieldName":"sessions" }, - { "type":"longMax", "name":"active_sessions", "fieldName":"active_sessions" }, - { "type":"longSum", "name":"closed_sessions", "fieldName":"closed_sessions" }, - { "type":"longSum", "name":"in_bytes", "fieldName":"in_bytes" }, - { "type":"longSum", "name":"out_bytes", "fieldName":"out_bytes" }, - { "type":"longSum", "name":"in_pkts", "fieldName":"in_pkts" }, - { "type":"longSum", "name":"out_pkts", "fieldName":"out_pkts" }, - { "type":"longSum", "name":"asymmetric_c2s_flows", "fieldName":"asymmetric_c2s_flows" }, - { "type":"longSum", "name":"asymmetric_s2c_flows", "fieldName":"asymmetric_s2c_flows" }, - { "type":"longSum", "name":"tcp_syn_pkts", "fieldName":"tcp_syn_pkts" }, - { "type":"longSum", "name":"tcp_ack_pkts", "fieldName":"tcp_ack_pkts" }, - { "type":"longSum", "name":"tcp_fin_pkts", "fieldName":"tcp_fin_pkts" }, - { "type":"longSum", "name":"tcp_rst_pkts", "fieldName":"tcp_rst_pkts" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1S"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, - {"type": "expression", "name": "active_sessions", "expression": "nvl(active_sessions, 0)"}, - {"type": "expression", "name": "closed_sessions", "expression": "nvl(closed_sessions, 0)"}, - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}, - {"type": "expression", "name": "asymmetric_c2s_flows", "expression": "nvl(asymmetric_c2s_flows, 0)"}, - {"type": "expression", "name": "asymmetric_s2c_flows", "expression": "nvl(asymmetric_s2c_flows, 0)"}, - {"type": "expression", "name": "tcp_syn_pkts", "expression": "nvl(tcp_syn_pkts, 0)"}, - {"type": "expression", "name": "tcp_ack_pkts", "expression": "nvl(tcp_ack_pkts, 0)"}, - {"type": "expression", "name": "tcp_fin_pkts", "expression": "nvl(tcp_fin_pkts, 0)"}, - {"type": "expression", "name": "tcp_rst_pkts", "expression": "nvl(tcp_rst_pkts, 0)"} - ], - "filter": { "type": "selector", "dimension": "name", "value": "traffic_general_stat" } - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 5000000, - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "NETWORK-TRAFFIC-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json deleted file mode 100644 index a5a644d..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "traffic_shaping_rule_hits", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "timestamp_ms", - "format": "millis" - }, - "dimensionsSpec": { - "dimensions": [ - {"name": "vsys_id","type": "long"}, - "device_id", - "device_group", - "data_center", - {"name": "rule_id","type": "long"}, - {"name": "profile_id","type": "long"}, - {"name": "priority","type": "long"} - ] - }, - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [ - { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, - { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, - { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, - { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, - { "name": "in_drop_pkts", "type": "path", "expr": "$.fields.in_drop_pkts" }, - { "name": "out_drop_pkts", "type": "path", "expr": "$.fields.out_drop_pkts" }, - { "name": "in_max_latency_us", "type": "path", "expr": "$.fields.in_max_latency_us" }, - { "name": "out_max_latency_us", "type": "path", "expr": "$.fields.out_max_latency_us" }, - { "name": "in_queue_len", "type": "path", "expr": "$.fields.in_queue_len" }, - { "name": "out_queue_len", "type": "path", "expr": "$.fields.out_queue_len" }, - { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, - { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, - { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, - { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, - { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, - { "name": "profile_id", "type": "path", "expr": "$.tags.profile_id" }, - { "name": "priority", "type": "path", "expr": "$.tags.priority" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, - { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, - { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, - { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }, - { "type" : "longSum", "name" : "in_drop_pkts", "fieldName" : "in_drop_pkts" }, - { "type" : "longSum", "name" : "out_drop_pkts", "fieldName" : "out_drop_pkts" }, - { "type" : "longMax", "name" : "in_max_latency_us", "fieldName" : "in_max_latency_us" }, - { "type" : "longMax", "name" : "out_max_latency_us", "fieldName" : "out_max_latency_us" }, - { "type" : "longMax", "name" : "in_queue_len", "fieldName" : "in_queue_len" }, - { "type" : "longMax", "name" : "out_queue_len", "fieldName" : "out_queue_len" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT1S"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, - {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, - {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, - {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}, - {"type": "expression", "name": "in_drop_pkts", "expression": "nvl(in_drop_pkts, 0)"}, - {"type": "expression", "name": "out_drop_pkts", "expression": "nvl(out_drop_pkts, 0)"}, - {"type": "expression", "name": "in_max_latency_us", "expression": "nvl(in_max_latency_us, 0)"}, - {"type": "expression", "name": "out_max_latency_us", "expression": "nvl(out_max_latency_us, 0)"}, - {"type": "expression", "name": "in_queue_len", "expression": "nvl(in_queue_len, 0)"}, - {"type": "expression", "name": "out_queue_len", "expression": "nvl(out_queue_len, 0)"} - ], - "filter": { "type": "selector", "dimension": "name", "value": "traffic_shaping_rule_hits" } - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "POLICY-RULE-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} - diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json deleted file mode 100644 index 5541452..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "type": "kafka", - "dataSchema": { - "dataSource": "traffic_top_destination_ip_metrics_log", - "parser": { - "type": "string", - "parseSpec": { - "format": "json", - "timestampSpec": { - "column": "sketch_start_time", - "format": "posix" - }, - "dimensionsSpec": { - "dimensions": [ - "destination_ip", - "attack_type", - { "name": "vsys_id", "type": "long" }, - { "type": "long", "name": "partition_num" } - ] - } - } - }, - "metricsSpec" : [ - { "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"}, - { "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" }, - { "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" } - ], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": {"type": "period", "period": "PT10M", "timeZone": "Asia/Hong_Kong"}, - "rollup": true - }, - "transformSpec" :{ - "transforms":[ - {"type": "expression","name": "vsys_id","expression": "nvl(vsys_id,1)"}, - {"type": "expression", "name": "session_rate", "expression": "nvl(session_rate, 0)"}, - {"type": "expression", "name": "packet_rate", "expression": "nvl(packet_rate, 0)"}, - {"type": "expression", "name": "bit_rate", "expression": "nvl(bit_rate, 0)"} - ] - } - }, - "tuningConfig": { - "type": "kafka", - "resetOffsetAutomatically": true, - "reportParseExceptions": false - }, - "ioConfig": { - "topic": "TRAFFIC-TOP-DESTINATION-IP-METRIC", - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT1H", - "completionTimeout": "PT30M", - "consumerProperties": { - "bootstrap.servers": "kafkabootstrap", - "sasl.mechanism": "PLAIN", - "security.protocol": "SASL_PLAINTEXT", - "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" - } - } -} diff --git a/tsg-olap/parcels/roles/initialization/druid/vars/main.yml b/tsg-olap/parcels/roles/initialization/druid/vars/main.yml deleted file mode 100644 index 3fc948a..0000000 --- a/tsg-olap/parcels/roles/initialization/druid/vars/main.yml +++ /dev/null @@ -1,11 +0,0 @@ -container_name: druid - -backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/{{ container_name }}" - -backup_items: - - conf - - docker-compose.yml - - init - - monitor - - topology - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml deleted file mode 100644 index 2c5336b..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml +++ /dev/null @@ -1,28 +0,0 @@ -- name: Create buckets - uri: - url: "http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/{{ item }}" - method: PUT - headers: - token: "{{ hos_token }}" - return_content: yes - retries: 3 - delay: 3 - loop: "{{ init_buckets }}" - run_once: true - -- name: Check if buckets exist - uri: - url: "http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/" - method: GET - headers: - token: "{{ hos_token }}" - return_content: yes - register: bucket_info - retries: 3 - delay: 3 - -- name: Check create bucket result - fail: - msg: "{{ item }} was not created successfully. Please retry." - when: "'{{ item }}' not in bucket_info.content" - loop: "{{ init_buckets }}" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml deleted file mode 100644 index 125d374..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -- include: init.yml - when: operation == 'install' - -- include: upgrade.yml - when: operation == 'upgrade' diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml deleted file mode 100644 index cd21505..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml deleted file mode 100644 index 9203abd..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -init_buckets: - - troubleshooting_file_bucket - - assessment_file_bucket - - knowledge_base_bucket - - traffic_file_bucket - - report_snapshot_bucket diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/mysql b/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/mysql deleted file mode 100644 index a4f1e76..0000000 Binary files a/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/mysql and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/schedule-service/dos-baseline/analyse/analyse-baseines.jar b/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/schedule-service/dos-baseline/analyse/analyse-baseines.jar deleted file mode 100644 index 0378b99..0000000 Binary files a/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/schedule-service/dos-baseline/analyse/analyse-baseines.jar and /dev/null differ diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml deleted file mode 100644 index 3a66bc3..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml +++ /dev/null @@ -1,51 +0,0 @@ -- name: Check if admin backup directory exists - stat: - path: "{{ backup_path }}/{{ admin_container_name }}" - register: exist_status - -- name: Backup directories - block: - - name: Creat backup directory - file: - state: directory - path: "{{ backup_path }}/{{ admin_container_name }}" - - - name: Copying directories to backup - copy: - src: "{{ deploy_dir }}/{{ admin_container_name }}/{{ item }}" - dest: "{{ backup_path }}/{{ admin_container_name }}/" - remote_src: true - loop: "{{ backup_items }}" - - rescue: - - name: Remove backup directory on failure - file: - path: "{{ backup_path }}/{{ admin_container_name }}" - state: absent - when: exist_status.stat.exists == false - -- name: Check if executor backup directory exists - stat: - path: "{{ backup_path }}/{{ executor_container_name }}" - register: exist_status - -- name: Backup directories - block: - - name: Creat backup directory - file: - state: directory - path: "{{ backup_path }}/{{ executor_container_name }}" - - - name: Copying directories to backup - copy: - src: "{{ deploy_dir }}/{{ executor_container_name }}/{{ item }}" - dest: "{{ backup_path }}/{{ executor_container_name }}/" - remote_src: true - loop: "{{ backup_items }}" - - rescue: - - name: Remove backup directory on failure - file: - path: "{{ backup_path }}/{{ executor_container_name }}" - state: absent - when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml deleted file mode 100644 index 14e1d40..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml +++ /dev/null @@ -1,12 +0,0 @@ -- name: Check the health status of galaxy-job-admin - uri: - url: http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin/login - method: POST - body_format: form-urlencoded - body: - userName: admin - pin: galaxy_2019 - -- name: Check that the galaxy-job-admin is accessible - uri: - url: "http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin/testCase/logPolicyTest" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml deleted file mode 100644 index 14cfe28..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml +++ /dev/null @@ -1,52 +0,0 @@ -- name: Set container_name - set_fact: - container_name: "{{ admin_container_name }}" - image_name: "{{ admin_image_name }}" - image_tag: "{{ admin_image_tag }}" - -- name: Creating directory - file: - state: directory - path: '{{ deploy_dir }}/{{ container_name }}/config' - -- name: Copying image to {{ deploy_dir }}/{{ container_name }}/ - copy: - src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' - dest: '{{ deploy_dir }}/{{ container_name }}/' - force: true - notify: - - Load Image - - Start Container - -- name: Copying {{ container_name }} config - template: - src: "admin/{{ item }}.j2" - dest: '{{ deploy_dir }}/{{ container_name }}/config/{{ item }}' - force: yes - loop: - - application.properties - - logback.xml - notify: - - Start Container - -- name: Copying {{ container_name }} docker-compose.yml - template: - src: "admin/docker-compose.yml.j2" - dest: '{{ deploy_dir }}/{{ container_name }}/docker-compose.yml' - force: yes - notify: - - Start Container - -- name: Copying {{ container_name }} nginx config - template: - src: "admin/{{ container_name }}.conf.j2" - dest: '{{ deploy_dir }}/galaxy-gateway-nginx/conf/includes/{{ container_name }}.conf' - force: yes - notify: - - Start Nginx - delegate_to: "{{ item }}" - run_once: true - loop: "{{ groups.loadbalancer[:2] }}" - -- meta: flush_handlers - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml deleted file mode 100644 index 7ea6d93..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml +++ /dev/null @@ -1,45 +0,0 @@ -- name: Setting variable name - set_fact: - container_name: "{{ executor_container_name }}" - image_name: "{{ executor_image_name }}" - image_tag: "{{ executor_image_tag }}" - -- name: Creating {{ container_name }} directory - file: - state: directory - path: '{{ deploy_dir }}/{{ container_name }}/{{ item }}' - loop: - - "init" - - "config" - -- name: Copying image to {{ deploy_dir }}/{{ container_name }}/ - copy: - src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' - dest: '{{ deploy_dir }}/{{ container_name }}/' - force: true - notify: - - Load Image - - Start Container - -- name: Copying {{ container_name }} config - template: - src: "executor/{{ item }}.j2" - dest: '{{ deploy_dir }}/{{ container_name }}/config/{{ item }}' - force: yes - loop: - - application.properties - - logback.xml - notify: - - Start Container - -- name: Copying {{ container_name }} docker-compose.yml - template: - src: "executor/docker-compose.yml.j2" - dest: '{{ deploy_dir }}/{{ container_name }}/docker-compose.yml' - force: yes - notify: - - Start Container - - -- meta: flush_handlers - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml deleted file mode 100644 index d3385f7..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml +++ /dev/null @@ -1,53 +0,0 @@ -- name: Creating directory - file: - state: directory - path: '{{ deploy_dir }}/{{ item }}/init' - loop: - - 'galaxy-job-admin' - - 'galaxy-job-executor' - -- name: Copying admin init files - template: - src: "{{ item.src }}" - dest: "{{ deploy_dir }}/{{ admin_container_name }}/init/{{ item.dest }}" - mode: "{{ item.mode }}" - force: true - loop: - - { src: "push_config.sh.j2", dest: "push_config.sh", mode: "0755" } - - { src: "admin/xxl_job.sql.j2", dest: "xxl_job.sql", mode: "0644" } - - { src: 'admin/create_mariadb_tables.sh.j2', dest: 'create_mariadb_tables.sh', mode: "0755" } - - { src: "admin/galaxy-job-admin.properties.j2", dest: "galaxy-job-admin.properties", mode: "0644" } - -- name: Copying mysql Client to /usr/bin/ - copy: - src: 'files/mysql' - dest: '/usr/bin/' - mode: 0755 - force: true - -- name: Creating table and init jobs - shell: cd {{ deploy_dir }}/{{ admin_container_name }}/init && sh create_mariadb_tables.sh - run_once: true - when: operation == 'install' - -- name: Copying executor init files - template: - src: "{{ item.src }}" - dest: "{{ deploy_dir }}/{{ executor_container_name }}/init/{{ item.dest }}" - mode: "{{ item.mode }}" - force: true - loop: - - { src: "push_config.sh.j2", dest: "push_config.sh", mode: "0755" } - - { src: "executor/galaxy-job-executor.properties.j2", dest: "galaxy-job-executor.properties", mode: "0644" } - -- name: Pushing Galaxy-job-service admin config - shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && sh push_config.sh {{ item.conf }}' - register: post_result - until: post_result.stdout == '1' - retries: 3 - delay: 3 - failed_when: post_result.stdout != '1' - loop: - - { src: "{{ admin_container_name }}" , conf: "galaxy-job-admin.properties" } - - { src: "{{ executor_container_name }}" , conf: "galaxy-job-executor.properties" } - run_once: true diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml deleted file mode 100644 index a1d5b98..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -- block: - - include: uninstall-admin.yml - - include: uninstall-executor.yml - - include: init.yml - - include: deploy-admin.yml - - include: deploy-executor.yml - when: operation == 'install' - -- block: - - include: check.yml - - include: backup.yml - - include: init.yml - - include: upgrade.yml - - include: deploy-executor.yml - - include: check.yml - when: operation == 'upgrade' - -- block: - - include: uninstall-admin.yml - - include: uninstall-executor.yml - when: operation == 'uninstall' - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml deleted file mode 100644 index 2ebe9c8..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml +++ /dev/null @@ -1,18 +0,0 @@ -- name: Stopping and removing {{ admin_container_name }} container - docker_container: - name: '{{ admin_container_name }}' - state: absent - force_kill: true - -- name: Removing old {{ admin_image_name }} image - docker_image: - name: '{{ admin_image_name }}' - tag: '{{ admin_image_tag }}' - state: absent - force_absent: true - -- name: Ansible delete old {{ deploy_dir }}/{{ admin_container_name }} - file: - path: '{{ deploy_dir }}/{{ admin_container_name }}' - state: absent - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml deleted file mode 100644 index 3e4deb8..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml +++ /dev/null @@ -1,18 +0,0 @@ -- name: Stopping and removing {{ executor_container_name }} container - docker_container: - name: '{{ executor_container_name }}' - state: absent - force_kill: true - -- name: Removing old {{ executor_image_name }} image - docker_image: - name: '{{ executor_image_name }}' - tag: '{{ executor_image_tag }}' - state: absent - force_absent: true - -- name: Ansible delete old {{ deploy_dir }}/{{ executor_container_name }} - file: - path: '{{ deploy_dir }}/{{ executor_container_name }}' - state: absent - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml deleted file mode 100644 index 5da8917..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml +++ /dev/null @@ -1,54 +0,0 @@ -- name: Creating config tmp directory - file: - state: directory - path: "/tmp/{{ item }}" - loop: - - "{{ admin_container_name }}" - - "{{ executor_container_name }}" - -- name: Creating upgrade directory - file: - state: directory - path: "{{ deploy_dir }}/{{ executor_container_name }}/upgrade/upgrade-sql" - -- name: Fetching config files - fetch: - src: "{{ deploy_dir }}/{{ item.name }}/{{ item.config }}" - dest: "/tmp/{{ item.name }}/{{ item.config }}" - flat: yes - loop: - - { name: "{{ admin_container_name }}", config: 'docker-compose.yml' } - - { name: "{{ executor_container_name }}", config: 'docker-compose.yml' } - -- name: Loading variable by admin docker-compose.yml - include_vars: - file: "/tmp/{{ admin_container_name }}/docker-compose.yml" - name: admin_docker_config - -- name: Loading variable by executor docker-compose.yml - include_vars: - file: "/tmp/{{ executor_container_name }}/docker-compose.yml" - name: executor_docker_config - -- name: Set variable - set_fact: - galaxy_job_service: - admin_java_opts: "{{ admin_docker_config['services']['galaxy-job-admin']['environment']['JAVA_OPTS'] }}" - executor_java_opts: "{{ executor_docker_config['services']['galaxy-job-executor']['environment']['JAVA_OPTS'] }}" - -- name: Copying upgrade sql file - copy: - src: 'files/upgrade-sql' - dest: '{{ deploy_dir }}/{{ executor_container_name }}/upgrade/' - force: true - -- name: Copying upgrade-mariadb-data.sh template - template: - src: "upgrade-mariadb-data.sh.j2" - dest: "{{ deploy_dir }}/{{ executor_container_name }}/upgrade/upgrade-mariadb-data.sh" - mode: 0755 - force: true - -- name: Upgrade mariadb data - shell: cd {{ deploy_dir }}/{{ executor_container_name }}/upgrade && sh upgrade-mariadb-data.sh - run_once: true diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 deleted file mode 100644 index 6ea1881..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 +++ /dev/null @@ -1,23 +0,0 @@ -#配置集的配置格式 -nacos.config.type=properties -#配置中心地址 -nacos.config.server-addr={{ vrrp_instance.default.virtual_ipaddress }}:8848 -#命名空间 -nacos.config.namespace={{ config_namespace }} -#数据集ID -nacos.config.data-id=galaxy-job-admin.properties -#开启自动刷新 -nacos.config.auto-refresh=true -#配置对应的分组 -nacos.config.group=Galaxy -#Nacos认证用户 -nacos.config.username=nacos -#Nacos认证密码 -nacos.config.password=nacos -#开启配置预加载功能 -nacos.config.bootstrap.enable=true -#开启Nacos支持日志级别的加载时机 -nacos.config.bootstrap.log-enable=true - -## log config -logging.config=./config/logback.xml diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 deleted file mode 100644 index 392997f..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} -e "create database if not exists {{ mariadb_job_database }} default character set utf8mb4 collate utf8mb4_general_ci;" - -mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} {{ mariadb_job_database }} < {{ deploy_dir }}/{{ admin_container_name }}/init/xxl_job.sql - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 deleted file mode 100644 index be4abb8..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 +++ /dev/null @@ -1,15 +0,0 @@ -version: '3' -services: - galaxy-job-admin: - image: {{ image_name }}:{{ image_tag }} - container_name: galaxy-job-admin - environment: - JAVA_OPTS: "{{ galaxy_job_service.admin_java_opts }}" - ports: - - "8181:8181" - volumes: - - "{{ deploy_dir }}/{{ container_name }}/logs:/logs" - - "{{ deploy_dir }}/{{ container_name }}/data:/data" - - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-admin/config" - restart: always - network_mode: "host" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 deleted file mode 100644 index bee9eea..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 +++ /dev/null @@ -1,16 +0,0 @@ -upstream jobAdmin { - -{% for host in groups['galaxy_job_service'][:2] %} - server {{ host }}:8184; -{% endfor %} - -} - -server { - listen 8181; - server_name localhost; - location / { - proxy_pass http://jobAdmin; - } -} - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 deleted file mode 100644 index 082e719..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 +++ /dev/null @@ -1,73 +0,0 @@ -### web -server.port=8184 -server.servlet.context-path=/xxl-job-admin -spring.application.name=galaxy-job-admin -### actuator -management.server.servlet.context-path=/actuator -management.health.mail.enabled=false -management.endpoints.web.exposure.include=* -#详细的应用健康信息 prometheus -management.endpoint.health.show-details=always - -management.endpoint.metrics.enabled=true -management.endpoint.prometheus.enabled=true -management.metrics.export.prometheus.enabled=true -management.metrics.tags.application=${spring.application.name} -management.metrics.tags.module=${spring.application.name} - - -### resources -spring.mvc.servlet.load-on-startup=0 -spring.mvc.static-path-pattern=/static/** -spring.resources.static-locations=classpath:/static/ - -### freemarker -spring.freemarker.templateLoaderPath=classpath:/templates/ -spring.freemarker.suffix=.ftl -spring.freemarker.charset=UTF-8 -spring.freemarker.request-context-attribute=request -spring.freemarker.settings.number_format=0.########## - -### mybatis -mybatis.mapper-locations=classpath:/mybatis-mapper/*Mapper.xml -#mybatis.type-aliases-package=com.xxl.job.admin.core.model - -### xxl-job, datasource -spring.datasource.url=jdbc:mysql://{{ vrrp_instance.default.virtual_ipaddress }}:3306/{{ mariadb_job_database }}?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&serverTimezone=UTC -spring.datasource.username=ENC(63aTpwv2vH0vPikW+3Jjig==) -spring.datasource.password=ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y) -spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver - -### datasource-pool -spring.datasource.type=com.zaxxer.hikari.HikariDataSource -spring.datasource.hikari.minimum-idle=10 -spring.datasource.hikari.maximum-pool-size=100 -spring.datasource.hikari.auto-commit=true -spring.datasource.hikari.idle-timeout=30000 -spring.datasource.hikari.pool-name=HikariCP -spring.datasource.hikari.max-lifetime=900000 -spring.datasource.hikari.connection-timeout=30000 -spring.datasource.hikari.connection-test-query=SELECT 1 - -### xxl-job, email -spring.mail.host=smtp.qq.com -spring.mail.port=25 -spring.mail.username=xxx@qq.com -spring.mail.password=xxx -spring.mail.properties.mail.smtp.auth=true -spring.mail.properties.mail.smtp.starttls.enable=true -spring.mail.properties.mail.smtp.starttls.required=true -spring.mail.properties.mail.smtp.socketFactory.class=javax.net.ssl.SSLSocketFactory - -### xxl-job, access token -xxl.job.accessToken= - -### xxl-job, i18n (default is zh_CN, and you can choose "zh_CN", "zh_TC" and "en") -xxl.job.i18n=zh_CN - -## xxl-job, triggerpool max size -xxl.job.triggerpool.fast.max=200 -xxl.job.triggerpool.slow.max=100 - -### xxl-job, log retention days -xxl.job.logretentiondays=30 diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 deleted file mode 100644 index dd729b0..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - - - - - - - ${LOG_PATTERN} - - - - - ${LOG_PATH}/${LOG_FILE_NAME}.log - - ALL - - - ${LOG_PATTERN} - - - - ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz - - ${LOG_DAYS} - ${LOG_SIZE} - - - - - - - - - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 deleted file mode 100644 index f3562c8..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 +++ /dev/null @@ -1,195 +0,0 @@ -/* -Navicat MySQL Data Transfer - -Target Server Type : MYSQL -Target Server Version : 50505 -File Encoding : 65001 - -Date: 2021-09-23 14:05:11 -*/ - -SET FOREIGN_KEY_CHECKS=0; - --- ---------------------------- --- Table structure for `xxl_job_group` --- ---------------------------- -DROP TABLE IF EXISTS `xxl_job_group`; -CREATE TABLE `xxl_job_group` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `app_name` varchar(64) NOT NULL COMMENT '执行器AppName', - `title` varchar(64) NOT NULL COMMENT '执行器名称', - `address_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '执行器地址类型:0=自动注册、1=手动录入', - `address_list` varchar(512) DEFAULT NULL COMMENT '执行器地址列表,多地址逗号分隔', - PRIMARY KEY (`id`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; - --- ---------------------------- --- Table structure for `xxl_job_lock` --- ---------------------------- -DROP TABLE IF EXISTS `xxl_job_lock`; -CREATE TABLE `xxl_job_lock` ( - `lock_name` varchar(50) NOT NULL COMMENT '锁名称', - PRIMARY KEY (`lock_name`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; - --- ---------------------------- --- Records of xxl_job_lock --- ---------------------------- -INSERT INTO `xxl_job_lock` VALUES ('schedule_lock'); - --- ---------------------------- --- Table structure for `xxl_job_log` --- ---------------------------- -DROP TABLE IF EXISTS `xxl_job_log`; -CREATE TABLE `xxl_job_log` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `job_group` int(11) NOT NULL COMMENT '执行器主键ID', - `job_id` int(11) NOT NULL COMMENT '任务,主键ID', - `executor_address` varchar(255) DEFAULT NULL COMMENT '执行器地址,本次执行的地址', - `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', - `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', - `executor_sharding_param` varchar(20) DEFAULT NULL COMMENT '执行器任务分片参数,格式如 1/2', - `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', - `trigger_time` datetime DEFAULT NULL COMMENT '调度-时间', - `trigger_code` int(11) NOT NULL COMMENT '调度-结果', - `trigger_msg` text DEFAULT NULL COMMENT '调度-日志', - `handle_time` datetime DEFAULT NULL COMMENT '执行-时间', - `handle_code` int(11) NOT NULL COMMENT '执行-状态', - `handle_msg` text DEFAULT NULL COMMENT '执行-日志', - `alarm_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '告警状态:0-默认、1-无需告警、2-告警成功、3-告警失败', - PRIMARY KEY (`id`) USING BTREE, - KEY `I_trigger_time` (`trigger_time`) USING BTREE, - KEY `I_handle_code` (`handle_code`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=2077229 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; - --- ---------------------------- --- Records of xxl_job_log --- ---------------------------- - --- ---------------------------- --- Table structure for `xxl_job_logglue` --- ---------------------------- -DROP TABLE IF EXISTS `xxl_job_logglue`; -CREATE TABLE `xxl_job_logglue` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `job_id` int(11) NOT NULL COMMENT '任务,主键ID', - `glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型', - `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', - `glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注', - `add_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; - --- ---------------------------- --- Records of xxl_job_logglue --- ---------------------------- - --- ---------------------------- --- Table structure for `xxl_job_log_report` --- ---------------------------- -DROP TABLE IF EXISTS `xxl_job_log_report`; -CREATE TABLE `xxl_job_log_report` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `trigger_day` datetime DEFAULT NULL COMMENT '调度-时间', - `running_count` int(11) NOT NULL DEFAULT 0 COMMENT '运行中-日志数量', - `suc_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行成功-日志数量', - `fail_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行失败-日志数量', - PRIMARY KEY (`id`) USING BTREE, - UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=786 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; - --- ---------------------------- --- Records of xxl_job_log_report --- ---------------------------- - --- ---------------------------- --- Table structure for `xxl_job_registry` --- ---------------------------- -DROP TABLE IF EXISTS `xxl_job_registry`; -CREATE TABLE `xxl_job_registry` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `registry_group` varchar(50) NOT NULL, - `registry_key` varchar(255) NOT NULL, - `registry_value` varchar(255) NOT NULL, - `update_time` datetime DEFAULT NULL, - PRIMARY KEY (`id`) USING BTREE, - KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=276 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; - --- ---------------------------- --- Table structure for `xxl_job_user` --- ---------------------------- -DROP TABLE IF EXISTS `xxl_job_user`; -CREATE TABLE `xxl_job_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `username` varchar(50) NOT NULL COMMENT '账号', - `password` varchar(50) NOT NULL COMMENT '密码', - `role` tinyint(4) NOT NULL COMMENT '角色:0-普通用户、1-管理员', - `permission` varchar(255) DEFAULT NULL COMMENT '权限:执行器ID列表,多个逗号分割', - PRIMARY KEY (`id`) USING BTREE, - UNIQUE KEY `i_username` (`username`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; - --- ---------------------------- --- Records of xxl_job_user --- ---------------------------- -INSERT INTO `xxl_job_user` VALUES ('1', 'admin', 'fea191a3fdd9f68503f9fa0e8d0293ab', '1', null); -INSERT INTO `xxl_job_user` VALUES ('2', 'query', '95497ea23bf8d27d86526717578e366f', '0', '3'); - --- ---------------------------- --- Table structure for `xxl_job_info` --- ---------------------------- -DROP TABLE IF EXISTS `xxl_job_info`; -CREATE TABLE `xxl_job_info` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `job_group` int(11) NOT NULL COMMENT '执行器主键ID', - `job_cron` varchar(128) NOT NULL COMMENT '任务执行CRON', - `job_desc` varchar(255) NOT NULL, - `add_time` datetime DEFAULT NULL, - `update_time` datetime DEFAULT NULL, - `author` varchar(64) DEFAULT NULL COMMENT '作者', - `alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件', - `executor_route_strategy` varchar(50) DEFAULT NULL COMMENT '执行器路由策略', - `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', - `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', - `executor_block_strategy` varchar(50) DEFAULT NULL COMMENT '阻塞处理策略', - `executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT '任务执行超时时间,单位秒', - `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', - `glue_type` varchar(50) NOT NULL COMMENT 'GLUE类型', - `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', - `glue_remark` varchar(128) DEFAULT NULL COMMENT 'GLUE备注', - `glue_updatetime` datetime DEFAULT NULL COMMENT 'GLUE更新时间', - `child_jobid` varchar(255) DEFAULT NULL COMMENT '子任务ID,多个逗号分隔', - `trigger_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '调度状态:0-停止,1-运行', - `trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上次调度时间', - `trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '下次调度时间', - PRIMARY KEY (`id`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=88 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; - - --- ---------------------------- --- Records of xxl_job_group --- ---------------------------- -INSERT INTO `xxl_job_group` VALUES (1, 'galaxy-executor', 'GDP', 1, '{{ job_executor_servers }}'); -INSERT INTO `xxl_job_group` VALUES (2, 'galaxy-executor', 'TSG_OLAP', 1, '{{ job_executor_servers }}'); --- ---------------------------- --- Records of xxl_job_info --- ---------------------------- -INSERT INTO `xxl_job_info` VALUES (58, 1, '0 0 0 1/1 * ? ', 'set_druid_table_ttl', '2020-07-30 10:34:34', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteReportAndMetricsDataJobHandler', '{\"defaultMaxDays\":365,\"maxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 10:34:34', '', 1, 1704326400000, 1704412800000); -INSERT INTO `xxl_job_info` VALUES (59, 1, '0 0 0 1 * ?', 'clear_druid_table', '2020-07-30 11:32:08', '2022-04-12 02:38:39', 'zhq', '', 'FAILOVER', 'deleteAllReportAndMetricsDataJobHandler', '{\"maxDays\":88,\"defaultMaxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:32:08', '', 0, 0, 0); -INSERT INTO `xxl_job_info` VALUES (62, 1, '0 0 0 1/1 * ? ', 'set_hos_bucket_ttl', '2020-07-30 11:59:30', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteFilesJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:30', '', 1, 1704326400000, 1704412800000); -INSERT INTO `xxl_job_info` VALUES (63, 1, '0 0 0 1 * ?', 'clear_hos_file', '2020-07-30 11:59:43', '2023-10-26 10:43:46', 'zhq', '', 'FAILOVER', 'deleteAllFilesJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:43', '', 0, 0, 0); -INSERT INTO `xxl_job_info` VALUES (64, 1, '0 0 0 1 * ?', 'clear_clickhouse_table', '2020-07-30 13:47:25', '2022-04-12 02:38:39', 'zhq', '', 'FAILOVER', 'deleteAllTrafficDataJobHandler', '{\"maxDays\":77,\"defaultMaxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:25', '', 0, 0, 0); -INSERT INTO `xxl_job_info` VALUES (65, 1, '0 0 0 1/1 * ? ', 'set_clickhouse_table_ttl', '2020-07-30 13:47:38', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteTrafficDataJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:38', '', 1, 1704326400000, 1704412800000); - -INSERT INTO `xxl_job_info` VALUES (29, 2, '0 0 4 * * ?', 'get_clickhoue_storage_daily_usage', '2020-03-26 10:58:09', '2021-09-09 11:54:53', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE\",\"ckDayGrowth\":\"true\"}', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-03-26 10:58:09', '', 1, 1704254400000, 1704340800000); -INSERT INTO `xxl_job_info` VALUES (36, 2, '0 0/5 * * * ?', 'get_database_storage_quota', '2020-04-07 21:41:20', '2021-09-09 11:54:44', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE\",\"ckDayGrowth\":\"false\"}', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-04-07 21:41:20', '', 1, 1704338100000, 1704338400000); -INSERT INTO `xxl_job_info` VALUES (86, 2, '0 0 1 * * ?', 'run_druid_segment_compaction', '2021-07-21 14:13:57', '2023-12-12 02:21:05', 'xpf', '', 'FAILOVER', 'druidCompactSegmentJobHandler', '[\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_fqdns\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_domains\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_client_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_external_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_internal_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_subscribers\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"security_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"monitor_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"proxy_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"application_protocol_stat\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_general_stat\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_top_destination_ip_metrics_log\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"service_chaining_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"service_function_status\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"statistics_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_shaping_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"object_statistics\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"statistics_rule\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\",\r\n \"maxRowsInMemory\": 500000,\r\n \"maxBytesInMemory\": -1\r\n },\r\n \"context\":{\r\n \"druid.indexer.runner.javaOptsArray\": [\"-Xms4g\", \"-Xmx4g\", \"-XX:MaxDirectMemorySize=2g\"]\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"sys_storage_log\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n }\r\n]', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2021-07-21 14:13:57', '', 1, 1704330000000, 1704416400000); -INSERT INTO `xxl_job_info` VALUES (101, 2, '0 0 0 1/7 * ?', 'run_web_sketch_assessment', '2022-01-27 17:40:42', '2022-02-23 13:34:51', 'galaxy', '', 'FAILOVER', 'httpToStoreJobHandler', '[{\r\n \"url\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/sql/?option=long_term&resultId=1&query=SELECT server_domain AS name, arrayStringConcat(groupUniqArray(decoded_as),\',\') AS protocol, count(*) AS sessions FROM tsg_galaxy_v3.session_record WHERE recv_time >= toStartOfDay(now()- INTERVAL 7 DAY) AND recv_time < toStartOfDay(now()) AND notEmpty(server_domain) GROUP BY name ORDER BY sessions DESC LIMIT 500000;\",\r\n \"method\": \"get\",\r\n \"requestBody\": {},\r\n \"resultKey\": \"data\",\r\n \"store\": \"all\"\r\n}, {\r\n \"url\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/sql/?option=long_term&resultId=1&query=SELECT server_domain AS name, arrayStringConcat(groupUniqArray(decoded_as),\',\') AS protocol, count(*) AS sessions FROM tsg_galaxy_v3.session_record WHERE recv_time >= toStartOfDay(now()- INTERVAL 7 DAY) AND recv_time < toStartOfDay(now()) AND notEmpty(server_domain) AND empty(fqdn_category_list) GROUP BY name ORDER BY sessions DESC LIMIT 500000;\",\r\n \"method\": \"get\",\r\n \"requestBody\": {},\r\n \"resultKey\": \"data\",\r\n \"store\": \"uncategorized\"\r\n}]', 'COVER_EARLY', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2022-01-27 17:40:42', '', 1, 1704067200000, 1704672000000); -INSERT INTO `xxl_job_info` VALUES (141, 2, '0 0 2 * * ?', 'set_clickhouse_table_and_field_ttl', '2022-06-07 07:42:38', '2022-06-08 09:28:19', 'wk', '', 'FIRST', 'changeCkTtlJobHandler', '', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2022-06-07 07:42:38', '', 1, 1704333600000, 1704420000000); -INSERT INTO `xxl_job_info` VALUES (143, 2, '0 0 0/1 * * ?', 'run_ip_learning', '2022-11-09 10:29:20', '2023-12-26 04:01:24', 'galaxy', '', 'FIRST', '', '', 'SERIAL_EXECUTION', 0, 0, 'GLUE_SHELL', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/iplearning/ && ./start.sh\'', '0001', '2023-12-26 04:01:24', '', 1, 1704337200000, 1704340800000); -INSERT INTO `xxl_job_info` VALUES (145, 2, '0 0 1 * * ?', 'run_dos_baseline', '2022-11-10 06:15:27', '2023-11-13 05:37:41', 'galaxy', '', 'FIRST', '', '', 'SERIAL_EXECUTION', 0, 0, 'GLUE_SHELL', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/dos-baseline && ./start.sh\'', '0000', '2023-11-13 05:37:41', '', 1, 1704330000000, 1704416400000); -INSERT INTO `xxl_job_info` VALUES (155, 2, '0 0 16 1/7 * ?', 'run_geoip_to_mmdb', '2022-12-02 06:56:18', '2023-12-21 09:44:44', 'galaxy', '', 'FAILOVER', 'ipLocateLibraryJobHandler', '{\r\n \"command\": \"docker run --rm --network=host -v {{ deploy_dir }}/schedule-service/mmdb:/mmdb --name perl-mmdb perl:v3.1 sh -c \'cd /mmdb && perl mmdb-perl.pl\' \",\r\n \"path\": \"/opt/schedule-service/mmdb\",\r\n \"knowledgeBaseServer\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base\",\r\n \"knowledgeBaseList\": [{\"name\": \"ip_builtin\",\"builtin\":1 ,\"kb_id\": \"64af7077-eb9b-4b8f-80cf-2ceebc89bea9\" },\r\n {\"name\": \"ip_user_defined\",\"builtin\":0 ,\"kb_id\": \"004390bc-3135-4a6f-a492-3662ecb9e289\" }]\r\n}', 'DISCARD_LATER', 0, 0, 'BEAN', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/dos-baseline && ./start.sh\'', 'GLUE代码初始化', '2022-12-02 06:56:18', '', 0, 0, 0); -INSERT INTO `xxl_job_info` VALUES (158, 2, '0 0 1 * * ?', 'run_druid_segement_granularity_compact', '2023-02-20 05:59:59', '2023-05-11 03:39:14', 'galaxy', '', 'FAILOVER', 'druidMergeHistoryDataJobHandler', '[\r\n {\r\n \"dataSource\": \"service_function_status\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n },\r\n {\r\n \"dataSource\": \"service_chaining_rule_hits\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n },\r\n {\r\n \"dataSource\": \"traffic_shaping_rule_hits\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n }\r\n]', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2023-02-20 05:59:59', '', 1, 1704330000000, 1704416400000); - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 deleted file mode 100644 index 205a6a0..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 +++ /dev/null @@ -1,23 +0,0 @@ -#配置集的配置格式 -nacos.config.type=properties -#配置中心地址 -nacos.config.server-addr={{ vrrp_instance.default.virtual_ipaddress }}:8848 -#命名空间 -nacos.config.namespace={{ config_namespace }} -#数据集ID -nacos.config.data-id=galaxy-job-executor.properties -#开启自动刷新 -nacos.config.auto-refresh=true -#配置对应的分组 -nacos.config.group=Galaxy -#Nacos认证用户 -nacos.config.username=nacos -#Nacos认证密码 -nacos.config.password=nacos -#开启配置预加载功能 -nacos.config.bootstrap.enable=true -#开启Nacos支持日志级别的加载时机 -nacos.config.bootstrap.log-enable=true - -### log config -logging.config=./config/logback.xml diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 deleted file mode 100644 index 1aab54c..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 +++ /dev/null @@ -1,19 +0,0 @@ -version: '3' -services: - galaxy-job-executor: - image: {{ image_name }}:{{ image_tag }} - container_name: galaxy-job-executor - environment: - JAVA_OPTS: "{{ galaxy_job_service.executor_java_opts }}" - ports: - - "8182:8182" - volumes: - - "{{ deploy_dir }}/{{ container_name }}/logs:/logs" - - "{{ deploy_dir }}/{{ container_name }}/data:/data" - - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-executor/config" - - "{{ deploy_dir }}/schedule-service:/opt/schedule-service" - - "/var/run/docker.sock:/var/run/docker.sock" - - "/usr/bin/docker:/usr/bin/docker" - restart: always - network_mode: "host" - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 deleted file mode 100644 index 5c8b0ef..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 +++ /dev/null @@ -1,65 +0,0 @@ -################################静态参数配置(修改后需要重启项目)################################ -### web port -server.port = 8185 -spring.application.name=galaxy-job-executor -### actuator -management.server.servlet.context-path=/actuator -management.health.mail.enabled=false -management.endpoints.web.exposure.include=* -#详细的应用健康信息 -management.endpoint.health.show-details=always -management.endpoint.metrics.enabled=true -management.endpoint.prometheus.enabled=true -management.metrics.export.prometheus.enabled=true -management.metrics.tags.application=${spring.application.name} -zookeeper.server={{ zookeeper_servers }} - -################################动态参数配置(修改后不需要重启项目)################################ -##存储配额文件服务器 -storge.files.hos-server={{ data_center_name }}|{{ vrrp_instance.oss.virtual_ipaddress }}:9098 -storge.files.token={{ hos_token }} -##存储配额查询druid -storge.analytic.server={{ data_center_name }}|{{ vrrp_instance.default.virtual_ipaddress }}:8089 -##存储配额查询clickhouse -storge.traffic.server={{ data_center_name }}|{{ vrrp_instance.default.virtual_ipaddress }}:8124 -storge.traffic.datasource=tsg_galaxy_v3 -storge.traffic.username=ENC(CNpgJlarVTNItkyW6DAQWA==) -storge.traffic.password=ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y) -#删除ttl -storge.traffic.system.parts=system.parts -#存储配额查询 -storge.traffic.system.partsclusters=system.parts_cluster -storge.traffic.system.disks=system.disks_cluster -storge.traffic.system.tables=system.tables_cluster -storge.traffic.system.clusters=system.clusters -#删除ttl白名单,多个逗号分隔 -storge.files.delete.exclusion=default,assessment_hos_bucket,knowledge_base_hos_bucket -storge.analytic.delete.exclusion=traffic_general_stat -storge.taffic.delete.exclusion=assessment_event_local -### xxl-job admin address list, such as "http://address" or "http://address01,http://address02" -xxl.job.admin.addresses={{ job_admin_servers }} -### xxl-job, access token -xxl.job.accessToken= -### xxl-job executor registry-address: default use address to registry , otherwise use ip:po -xxl.job.executor.appname=galaxy-executor -### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null -xxl.job.executor.address= -### xxl-job executor server-info -xxl.job.executor.ip= -xxl.job.executor.port=8886 -### xxl-job executor log-path -xxl.job.executor.logpath=/data/logs/jobhandler -### xxl-job executor log-retention-days -xxl.job.executor.logretentiondays=30 -## http pool config -### max connection number -http.pool.max.connection=500 -http.pool.request.timeout=120000 -http.pool.response.timeout=300000 -http.pool.max.per.route=300 -http.pool.connect.timeout=10000 -##指定kafka server的地址,集群配多个,中间,逗号隔开 -spring.kafka.bootstrap-servers={{ kafka_sink_servers }} -spring.kafka.ssl.enable=true -spring.kafka.ssl.username=ENC(RYN0it9o1vYynI1xlmX/Gw==) -spring.kafka.ssl.pin=ENC(vlb4F5Y3ff0Am/3MMPdIpYjZq8wcGNMd) diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 deleted file mode 100644 index 758d04d..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - ${LOG_PATTERN} - - - - ${LOG_PATH}/${LOG_FILE_NAME}.log - - ALL - - - ${LOG_PATTERN} - - - - ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz - - ${LOG_DAYS} - ${LOG_SIZE} - - - - - - - - - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 deleted file mode 100644 index 87689e5..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -config_path_root=$1 -nacos_server={{ vrrp_instance.default.virtual_ipaddress }} - -tenant="{{ config_namespace }}" -app_name="galaxy-job-service" - -shell_result=1 - -# 推送配置 -function push() { - local config_file=$1 - local data_id="${config_file##*/}" - local suffix="${config_file##*.}" - - [ $suffix == "sql" ] && suffix="text" - [ $suffix == "yml" ] && suffix="yaml" - - local nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy" - local result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content@"$config_file") - if [[ $result -eq '200' ]]; then - sleep 2 - check=$(curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id") - #echo "$check $data_id" - - if [[ $check -ne '200' ]]; then - shell_result=0 - fi - fi -} - -# 获取本地文件 -function pushConfig() { - for config_path in $(readlink -f $config_path_root); do - if [ -d $config_path ]; then - for file in $(ls $config_path); do - config_path_file=$config_path/$file - if [ -f $config_path_file ]; then - push $config_path_file - fi - done - fi - if [ -f $config_path ]; then - push $config_path - fi - done -} - -pushConfig -echo $shell_result - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 deleted file mode 100644 index bfd7c17..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -set -e - -# 数据库连接参数 -upgrade_dir=${1:-`pwd`/upgrade-sql} - -# 指定当前版本和历史版本 -current_version="{{ solution_version }}" -old_version="{{ old_version }}" - -# 数据库连接参数 -db_hostname="{{ vrrp_instance.default.virtual_ipaddress }}" -db_name="xxl_job" -db_username="root" -db_password="{{ mariadb_default_pin }}" - -suffix=".sql" -# 获取更新目录中的所有SQL文件 -sql_files=$(find "${upgrade_dir}" -name "*$suffix") - -# 标记是否开始执行历史版本的标识 -start_execution=false - -# 循环处理每个SQL文件 -for file in ${sql_files}; do - # 从文件名中提取版本号 - filename=$(basename "$file") - version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.') - current_version="${current_version//./}" - old_version="${old_version//./}" - # 检查是否达到历史版本 - if [[ "${version}" -gt "${old_version}" ]]; then - start_execution=true - fi - - # 检查是否开始执行历史版本之后的SQL文件 - if [[ ${start_execution} = true ]]; then - echo "Executing SQL file: ${file}" - - # 执行SQL文件 - mysql -u "${db_username}" -p"${db_password}" -h "${db_hostname}" "${db_name}" < "${file}" - echo "Upgrade ${version} successfully" - fi - - # 检查是否达到当前版本 - if [[ "${version}" -ge "${current_version}" ]]; then - break - fi -done - diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml deleted file mode 100644 index de010b3..0000000 --- a/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml +++ /dev/null @@ -1,26 +0,0 @@ -software_packages_dir: "{{ role_path }}/../software-packages" - -admin_image_name: galaxy-job-admin -executor_image_name: galaxy-job-executor - -admin_image_tag: v1.3.220308 -executor_image_tag: v1.3.231220 - -admin_container_name: galaxy-job-admin -executor_container_name: galaxy-job-executor - - -mariadb_job_database: xxl_job -job_executor_servers: "{% for host in groups['galaxy_job_service'] %}http://{{ host }}:8886/{% if not loop.last %},{% endif %}{% endfor %}" - -job_admin_servers: "{% for host in groups['galaxy_job_service'] %}http://{{ host }}:8184/xxl-job-admin{% if not loop.last %},{% endif %}{% endfor %}" - -kafka_sink_servers: "{% for host in groups['kafka'] %}{{ host }}:9094{% if not loop.last %},{% endif %}{% endfor %}" -zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}" - -backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/" - -backup_items: - - config - - docker-compose.yml - - init