From afdcfa58380d275c09bac53c5e8cfbb65ac144bb Mon Sep 17 00:00:00 2001 From: lifengchao Date: Mon, 22 Apr 2024 13:49:19 +0800 Subject: [PATCH] =?UTF-8?q?TSG-20773=20clickhouse=E5=85=AC=E5=85=B1?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E6=B7=BB=E5=8A=A0app=5Fextra=5Finfo(24-02?= =?UTF-8?q?=E7=89=88=E6=9C=AC)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../02_init_new_table.sql | 8405 +++++++++-------- .../TSG 2307版本到2402版本升级操作/03_check.sql | 10 +- .../TSG-24.02/clickhouse/check-24.02.sql | 10 +- .../TSG-24.02/clickhouse/update-24.02-ck.sql | 26 + 4 files changed, 4248 insertions(+), 4203 deletions(-) diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_init_new_table.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_init_new_table.sql index 99ccb2d..890f050 100644 --- a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_init_new_table.sql +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_init_new_table.sql @@ -1,4193 +1,4212 @@ -create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; -create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query; - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster ( - vsys_id Int32, - recv_time Int64, - log_id UInt64, - profile_id Int64, - start_time Int64, - end_time Int64, - attack_type String, - severity String, - conditions String, - destination_ip String, - destination_country String, - source_ip_list String, - source_country_list String, - session_rate Int64, - packet_rate Int64, - bit_rate Int64 -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,destination_ip,recv_time,log_id); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster ( - vsys_id Int32, - recv_time Int64, - log_id UInt64, - profile_id Int64, - start_time Int64, - end_time Int64, - attack_type String, - severity String, - conditions String, - destination_ip String, - destination_country String, - source_ip_list String, - source_country_list String, - session_rate Int64, - packet_rate Int64, - bit_rate Int64 -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_query ( - vsys_id Int32, - recv_time Int64, - log_id UInt64, - profile_id Int64, - start_time Int64, - end_time Int64, - attack_type String, - severity String, - conditions String, - destination_ip String, - destination_country String, - source_ip_list String, - source_country_list String, - session_rate Int64, - packet_rate Int64, - bit_rate Int64 -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster ( - log_id UInt64, - recv_time Int64, - vsys_id Int64, - assessment_date Int64, - lot_number String, - file_name String, - assessment_file String, - assessment_type String, - features String, - size Int64, - file_checksum_sha String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,recv_time,log_id); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query ( - log_id UInt64, - recv_time Int64, - vsys_id Int64, - assessment_date Int64, - lot_number String, - file_name String, - assessment_file String, - assessment_type String, - features String, - size Int64, - file_checksum_sha String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster ( - log_id UInt64, - recv_time Int64, - vsys_id Int64, - assessment_date Int64, - lot_number String, - file_name String, - assessment_file String, - assessment_type String, - features String, - size Int64, - file_checksum_sha String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); - - - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -ssl_version String, -ssl_sni String, -ssl_san String, -ssl_cn String, -ssl_handshake_latency_ms Nullable(Int32), -ssl_ja3_hash String, -ssl_ja3s_hash String, -ssl_cert_issuer String, -ssl_cert_subject String, -ssl_esni_flag Nullable(Int32), -ssl_ech_flag Nullable(Int32), -dtls_cookie String, -dtls_version  String, -dtls_sni String, -dtls_san String, -dtls_cn String, -dtls_handshake_latency_ms Nullable(Int32), -dtls_ja3_fingerprint String, -dtls_ja3_hash String, -dtls_cert_issuer String, -dtls_cert_subject String, -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -ftp_account String, -ftp_url String, -ftp_link_type String, -quic_version String, -quic_sni String, -quic_user_agent String, -rdp_cookie String, -rdp_security_protocol String, -rdp_client_channels String, -rdp_keyboard_layout String, -rdp_client_version String, -rdp_client_name String, -rdp_client_product_id String, -rdp_desktop_width String, -rdp_desktop_height String, -rdp_requested_color_depth String, -rdp_certificate_type String, -rdp_certificate_count Nullable(Int32), -rdp_certificate_permanent Nullable(Int32), -rdp_encryption_level String, -rdp_encryption_method String, -ssh_version String, -ssh_auth_success String, -ssh_client_version String, -ssh_server_version String, -ssh_cipher_alg String, -ssh_mac_alg String, -ssh_compression_alg String, -ssh_kex_alg String, -ssh_host_key_alg String, -ssh_host_key String, -ssh_hassh String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -stratum_cryptocurrency String, -stratum_mining_pools String, -stratum_mining_program String, -stratum_mining_subscribe String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -address_type Int32, -vsys_id Int32, -client_ip String, -client_port Int32, -server_ip String, -server_port Int32, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,session_id,recv_time); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -address_type Int32, -vsys_id Int32, -client_ip String, -client_port Int32, -server_ip String, -server_port Int32, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -ingestion_time Int64, -processing_time Int64, -insert_time Int64 , -address_type Int32, -vsys_id Int32, -client_ip String, -client_port Int32, -server_ip String, -server_port Int32, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -dns_message_id Nullable(Int32), -dns_qr Nullable(Int32), -dns_opcode Nullable(Int32), -dns_aa Nullable(Int32), -dns_tc Nullable(Int32), -dns_rd Nullable(Int32), -dns_ra Nullable(Int32), -dns_rcode Nullable(Int32), -dns_qdcount Nullable(Int32), -dns_ancount Nullable(Int32), -dns_nscount Nullable(Int32), -dns_arcount Nullable(Int32), -dns_qname String, -dns_qtype Nullable(Int32), -dns_qclass Nullable(Int32), -dns_cname String, -dns_sub Nullable(Int32), -dns_rr String, -dns_response_latency_ms Nullable(Int32), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -mail_protocol_type String, -mail_account String, -mail_from_cmd String, -mail_to_cmd String, -mail_from String, -mail_password String, -mail_to String, -mail_cc String, -mail_bcc String, -mail_subject String, -mail_subject_charset String, -mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), -mail_eml_file String, -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); - - -alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; -alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -sip_call_id String, -sip_originator_description String, -sip_responder_description String, -sip_user_agent String, -sip_server String, -sip_originator_sdp_connect_ip String, -sip_originator_sdp_media_port Nullable(Int32), -sip_originator_sdp_media_type String, -sip_originator_sdp_content String, -sip_responder_sdp_connect_ip String, -sip_responder_sdp_media_port Nullable(Int32), -sip_responder_sdp_media_type String, -sip_responder_sdp_content String, -sip_duration_s Nullable(Int32), -sip_bye String, -rtp_payload_type_c2s Nullable(Int32), -rtp_payload_type_s2c Nullable(Int32), -rtp_pcap_path String, -rtp_originator_dir Nullable(Int32), -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); - - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64 MATERIALIZED toUnixTimestamp(now()), -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -doh_url String, -doh_host String, -doh_request_line String, -doh_response_line String, -doh_cookie String, -doh_referer String, -doh_user_agent String, -doh_content_length String, -doh_content_type String, -doh_set_cookie String, -doh_version String, -doh_message_id Int64, -doh_qr Nullable(Int64), -doh_opcode Nullable(Int64), -doh_aa Nullable(Int64), -doh_tc Nullable(Int64), -doh_rd Nullable(Int64), -doh_ra Nullable(Int64), -doh_rcode Nullable(Int64), -doh_qdcount Nullable(Int64), -doh_ancount Nullable(Int64), -doh_nscount Nullable(Int64), -doh_arcount Nullable(Int64), -doh_qname String, -doh_qtype Nullable(Int64), -doh_qclass Nullable(Int64), -doh_cname String, -doh_sub Nullable(Int64), -doh_rr String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -doh_url String, -doh_host String, -doh_request_line String, -doh_response_line String, -doh_cookie String, -doh_referer String, -doh_user_agent String, -doh_content_length String, -doh_content_type String, -doh_set_cookie String, -doh_version String, -doh_message_id Int64, -doh_qr Nullable(Int64), -doh_opcode Nullable(Int64), -doh_aa Nullable(Int64), -doh_tc Nullable(Int64), -doh_rd Nullable(Int64), -doh_ra Nullable(Int64), -doh_rcode Nullable(Int64), -doh_qdcount Nullable(Int64), -doh_ancount Nullable(Int64), -doh_nscount Nullable(Int64), -doh_arcount Nullable(Int64), -doh_qname String, -doh_qtype Nullable(Int64), -doh_qclass Nullable(Int64), -doh_cname String, -doh_sub Nullable(Int64), -doh_rr String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_query ( -recv_time Int64, -log_id UInt64, -decoded_as String, -session_id UInt64, -start_timestamp_ms DateTime64(3), -end_timestamp_ms DateTime64(3), -duration_ms Int32, -tcp_handshake_latency_ms Nullable(Int32), -ingestion_time Int64, -processing_time Int64, -insert_time Int64, -device_id String, -out_link_id Nullable(Int32), -in_link_id Nullable(Int32), -device_tag String, -data_center String, -device_group String, -sled_ip String, -address_type Int32, -vsys_id Int32, -t_vsys_id Int32, -flags Int64, -flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), -client_ip String, -client_port Int32, -client_os_desc String, -client_geolocation LowCardinality(String), -client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, -server_ip String, -server_port Int32, -server_os_desc String, -server_geolocation LowCardinality(String), -server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -fqdn_category_list Array(Int64), -ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), -http_url String, -http_host String, -http_request_line String, -http_response_line String, -http_request_body String, -http_response_body String, -http_proxy_flag Nullable(Int32), -http_sequence Nullable(Int32), -http_cookie String, -http_referer String, -http_user_agent String, -http_request_content_length Nullable(Int64), -http_request_content_type String, -http_response_content_length Nullable(Int64), -http_response_content_type String, -http_set_cookie String, -http_version String, -http_status_code Nullable(Int32), -http_response_latency_ms Nullable(Int32), -http_session_duration_ms Nullable(Int32), -http_action_file_size Nullable(Int64), -doh_url String, -doh_host String, -doh_request_line String, -doh_response_line String, -doh_cookie String, -doh_referer String, -doh_user_agent String, -doh_content_length String, -doh_content_type String, -doh_set_cookie String, -doh_version String, -doh_message_id Int64, -doh_qr Nullable(Int64), -doh_opcode Nullable(Int64), -doh_aa Nullable(Int64), -doh_tc Nullable(Int64), -doh_rd Nullable(Int64), -doh_ra Nullable(Int64), -doh_rcode Nullable(Int64), -doh_qdcount Nullable(Int64), -doh_ancount Nullable(Int64), -doh_nscount Nullable(Int64), -doh_arcount Nullable(Int64), -doh_qname String, -doh_qtype Nullable(Int64), -doh_qclass Nullable(Int64), -doh_cname String, -doh_sub Nullable(Int64), -doh_rr String, -sent_pkts Int64, -received_pkts Int64, -sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); - - --- tsg_galaxy_v3.security_event_materialized_view -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster -TO tsg_galaxy_v3.security_event_local -( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_starttls_flag Nullable(Int32), - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -AS -SELECT - recv_time, - log_id, - decoded_as, - session_id, - start_timestamp_ms, - end_timestamp_ms, - duration_ms, - tcp_handshake_latency_ms, - ingestion_time, - processing_time, - -- insert_time, - device_id, - out_link_id, - in_link_id, - device_tag, - data_center, - device_group, - sled_ip, - address_type, - vsys_id, - t_vsys_id, - flags, - flags_identify_info, - security_rule_list, - security_action, - monitor_rule_list, - shaping_rule_list, - proxy_rule_list, - statistics_rule_list, - sc_rule_list, - sc_rsp_raw, - sc_rsp_decrypted, - proxy_action, - proxy_pinning_status, - proxy_intercept_status, - proxy_passthrough_reason, - proxy_client_side_latency_ms, - proxy_server_side_latency_ms, - proxy_client_side_version, - proxy_server_side_version, - proxy_cert_verify, - proxy_intercept_error, - monitor_mirrored_pkts, - monitor_mirrored_bytes, - client_ip, - client_port, - client_os_desc, - client_geolocation, - client_asn, - subscriber_id, - imei, - imsi, - phone_number, - apn, - server_ip, - server_port, - server_os_desc, - server_geolocation, - server_asn, - server_fqdn, - server_domain, - app_transition, - app, - app_debug_info, - app_content, - fqdn_category_list, - ip_protocol, - decoded_path, - dns_message_id, - dns_qr, - dns_opcode, - dns_aa, - dns_tc, - dns_rd, - dns_ra, - dns_rcode, - dns_qdcount, - dns_ancount, - dns_nscount, - dns_arcount, - dns_qname, - dns_qtype, - dns_qclass, - dns_cname, - dns_sub, - dns_rr, - dns_response_latency_ms, - http_url, - http_host, - http_request_line, - http_response_line, - http_request_body, - http_response_body, - http_proxy_flag, - http_sequence, - http_cookie, - http_referer, - http_user_agent, - http_request_content_length, - http_request_content_type, - http_response_content_length, - http_response_content_type, - http_set_cookie, - http_version, - http_status_code, - http_response_latency_ms, - http_session_duration_ms, - http_action_file_size, - ssl_version, - ssl_sni, - ssl_san, - ssl_cn, - ssl_handshake_latency_ms, - ssl_ja3_hash, - ssl_ja3s_hash, - ssl_cert_issuer, - ssl_cert_subject, - ssl_esni_flag, - ssl_ech_flag, - dtls_cookie, - dtls_version, - dtls_sni, - dtls_san, - dtls_cn, - dtls_handshake_latency_ms, - dtls_ja3_fingerprint, - dtls_ja3_hash, - dtls_cert_issuer, - dtls_cert_subject, - mail_protocol_type, - mail_account, - mail_from_cmd, - mail_to_cmd, - mail_from, - mail_password, - mail_to, - mail_cc, - mail_bcc, - mail_subject, - mail_subject_charset, - mail_attachment_name, - mail_attachment_name_charset, - mail_starttls_flag, - mail_eml_file, - ftp_account, - ftp_url, - ftp_link_type, - quic_version, - quic_sni, - quic_user_agent, - rdp_cookie, - rdp_security_protocol, - rdp_client_channels, - rdp_keyboard_layout, - rdp_client_version, - rdp_client_name, - rdp_client_product_id, - rdp_desktop_width, - rdp_desktop_height, - rdp_requested_color_depth, - rdp_certificate_type, - rdp_certificate_count, - rdp_certificate_permanent, - rdp_encryption_level, - rdp_encryption_method, - ssh_version, - ssh_auth_success, - ssh_client_version, - ssh_server_version, - ssh_cipher_alg, - ssh_mac_alg, - ssh_compression_alg, - ssh_kex_alg, - ssh_host_key_alg, - ssh_host_key, - ssh_hassh, - sip_call_id, - sip_originator_description, - sip_responder_description, - sip_user_agent, - sip_server, - sip_originator_sdp_connect_ip, - sip_originator_sdp_media_port, - sip_originator_sdp_media_type, - sip_originator_sdp_content, - sip_responder_sdp_connect_ip, - sip_responder_sdp_media_port, - sip_responder_sdp_media_type, - sip_responder_sdp_content, - sip_duration_s, - sip_bye, - rtp_payload_type_c2s, - rtp_payload_type_s2c, - rtp_pcap_path, - rtp_originator_dir, - stratum_cryptocurrency, - stratum_mining_pools, - stratum_mining_program, - stratum_mining_subscribe, - sent_pkts, - received_pkts, - sent_bytes, - received_bytes, - tcp_c2s_ip_fragments, - tcp_s2c_ip_fragments, - tcp_c2s_lost_bytes, - tcp_s2c_lost_bytes, - tcp_c2s_o3_pkts, - tcp_s2c_o3_pkts, - tcp_c2s_rtx_pkts, - tcp_s2c_rtx_pkts, - tcp_c2s_rtx_bytes, - tcp_s2c_rtx_bytes, - tcp_rtt_ms, - tcp_client_isn, - tcp_server_isn, - packet_capture_file, - in_src_mac, - out_src_mac, - in_dest_mac, - out_dest_mac, - encapsulation, - dup_traffic_flag, - tunnel_endpoint_a_desc, - tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.session_record_local -WHERE empty(security_rule_list) = 0 -; - --- tsg_galaxy_v3.monitor_event_materialized_view -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster -TO tsg_galaxy_v3.monitor_event_local -( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_starttls_flag Nullable(Int32), - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -AS -SELECT - recv_time, - log_id, - decoded_as, - session_id, - start_timestamp_ms, - end_timestamp_ms, - duration_ms, - tcp_handshake_latency_ms, - ingestion_time, - processing_time, - -- insert_time, - device_id, - out_link_id, - in_link_id, - device_tag, - data_center, - device_group, - sled_ip, - address_type, - vsys_id, - t_vsys_id, - flags, - flags_identify_info, - security_rule_list, - security_action, - monitor_rule_list, - shaping_rule_list, - proxy_rule_list, - statistics_rule_list, - sc_rule_list, - sc_rsp_raw, - sc_rsp_decrypted, - proxy_action, - proxy_pinning_status, - proxy_intercept_status, - proxy_passthrough_reason, - proxy_client_side_latency_ms, - proxy_server_side_latency_ms, - proxy_client_side_version, - proxy_server_side_version, - proxy_cert_verify, - proxy_intercept_error, - monitor_mirrored_pkts, - monitor_mirrored_bytes, - client_ip, - client_port, - client_os_desc, - client_geolocation, - client_asn, - subscriber_id, - imei, - imsi, - phone_number, - apn, - server_ip, - server_port, - server_os_desc, - server_geolocation, - server_asn, - server_fqdn, - server_domain, - app_transition, - app, - app_debug_info, - app_content, - fqdn_category_list, - ip_protocol, - decoded_path, - dns_message_id, - dns_qr, - dns_opcode, - dns_aa, - dns_tc, - dns_rd, - dns_ra, - dns_rcode, - dns_qdcount, - dns_ancount, - dns_nscount, - dns_arcount, - dns_qname, - dns_qtype, - dns_qclass, - dns_cname, - dns_sub, - dns_rr, - dns_response_latency_ms, - http_url, - http_host, - http_request_line, - http_response_line, - http_request_body, - http_response_body, - http_proxy_flag, - http_sequence, - http_cookie, - http_referer, - http_user_agent, - http_request_content_length, - http_request_content_type, - http_response_content_length, - http_response_content_type, - http_set_cookie, - http_version, - http_status_code, - http_response_latency_ms, - http_session_duration_ms, - http_action_file_size, - ssl_version, - ssl_sni, - ssl_san, - ssl_cn, - ssl_handshake_latency_ms, - ssl_ja3_hash, - ssl_ja3s_hash, - ssl_cert_issuer, - ssl_cert_subject, - ssl_esni_flag, - ssl_ech_flag, - dtls_cookie, - dtls_version, - dtls_sni, - dtls_san, - dtls_cn, - dtls_handshake_latency_ms, - dtls_ja3_fingerprint, - dtls_ja3_hash, - dtls_cert_issuer, - dtls_cert_subject, - mail_protocol_type, - mail_account, - mail_from_cmd, - mail_to_cmd, - mail_from, - mail_password, - mail_to, - mail_cc, - mail_bcc, - mail_subject, - mail_subject_charset, - mail_attachment_name, - mail_attachment_name_charset, - mail_starttls_flag, - mail_eml_file, - ftp_account, - ftp_url, - ftp_link_type, - quic_version, - quic_sni, - quic_user_agent, - rdp_cookie, - rdp_security_protocol, - rdp_client_channels, - rdp_keyboard_layout, - rdp_client_version, - rdp_client_name, - rdp_client_product_id, - rdp_desktop_width, - rdp_desktop_height, - rdp_requested_color_depth, - rdp_certificate_type, - rdp_certificate_count, - rdp_certificate_permanent, - rdp_encryption_level, - rdp_encryption_method, - ssh_version, - ssh_auth_success, - ssh_client_version, - ssh_server_version, - ssh_cipher_alg, - ssh_mac_alg, - ssh_compression_alg, - ssh_kex_alg, - ssh_host_key_alg, - ssh_host_key, - ssh_hassh, - sip_call_id, - sip_originator_description, - sip_responder_description, - sip_user_agent, - sip_server, - sip_originator_sdp_connect_ip, - sip_originator_sdp_media_port, - sip_originator_sdp_media_type, - sip_originator_sdp_content, - sip_responder_sdp_connect_ip, - sip_responder_sdp_media_port, - sip_responder_sdp_media_type, - sip_responder_sdp_content, - sip_duration_s, - sip_bye, - rtp_payload_type_c2s, - rtp_payload_type_s2c, - rtp_pcap_path, - rtp_originator_dir, - stratum_cryptocurrency, - stratum_mining_pools, - stratum_mining_program, - stratum_mining_subscribe, - sent_pkts, - received_pkts, - sent_bytes, - received_bytes, - tcp_c2s_ip_fragments, - tcp_s2c_ip_fragments, - tcp_c2s_lost_bytes, - tcp_s2c_lost_bytes, - tcp_c2s_o3_pkts, - tcp_s2c_o3_pkts, - tcp_c2s_rtx_pkts, - tcp_s2c_rtx_pkts, - tcp_c2s_rtx_bytes, - tcp_s2c_rtx_bytes, - tcp_rtt_ms, - tcp_client_isn, - tcp_server_isn, - packet_capture_file, - in_src_mac, - out_src_mac, - in_dest_mac, - out_dest_mac, - encapsulation, - dup_traffic_flag, - tunnel_endpoint_a_desc, - tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.session_record_local -WHERE empty(monitor_rule_list) = 0 -; +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query; + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,destination_ip,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_query ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,session_id,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + +-- tsg_galaxy_v3.security_event_materialized_view +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.security_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app_transition String, + app LowCardinality(String), + app_debug_info String, + app_content String, + app_extra_info String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_starttls_flag Nullable(Int32), + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + encapsulation String, + dup_traffic_flag Nullable(Int32), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_port, + server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + app_debug_info, + app_content, + app_extra_info, + fqdn_category_list, + ip_protocol, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_starttls_flag, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(security_rule_list) = 0 +; + +-- tsg_galaxy_v3.monitor_event_materialized_view +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.monitor_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app_transition String, + app LowCardinality(String), + app_debug_info String, + app_content String, + app_extra_info String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_starttls_flag Nullable(Int32), + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + encapsulation String, + dup_traffic_flag Nullable(Int32), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_port, + server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + app_debug_info, + app_content, + app_extra_info, + fqdn_category_list, + ip_protocol, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_starttls_flag, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(monitor_rule_list) = 0 +; diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_check.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_check.sql index baba079..f208416 100644 --- a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_check.sql +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_check.sql @@ -2,17 +2,17 @@ SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, asses FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time