From 2919f4e94b51a4fb4c5a62397fd59bb54ede443c Mon Sep 17 00:00:00 2001 From: houjinchuan Date: Wed, 28 Dec 2022 10:31:15 +0800 Subject: [PATCH] =?UTF-8?q?tsg=20ck=2023.01=E6=96=B0=E5=A2=9E=E6=9B=B4?= =?UTF-8?q?=E6=94=B9=E8=A1=A8=E7=BB=93=E6=9E=84=E5=90=8E=E7=9A=84=E5=BB=BA?= =?UTF-8?q?=E8=A1=A8=E8=AF=AD=E5=8F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Clickhouse_TSG_建表语句_NEW.sql | 5082 +++++++++++++++++ ...G_建表语句.sql => Clickhouse_TSG_建表语句_OLD.sql} | 69 +- 2 files changed, 5139 insertions(+), 12 deletions(-) create mode 100644 Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_NEW.sql rename Clickhouse最新全量建表语句/{Clickhouse_TSG_建表语句.sql => Clickhouse_TSG_建表语句_OLD.sql} (98%) diff --git a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_NEW.sql b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_NEW.sql new file mode 100644 index 0000000..9ff4e6b --- /dev/null +++ b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_NEW.sql @@ -0,0 +1,5082 @@ +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local ON CLUSTER ck_cluster +( + common_log_id UInt64, + common_recv_time Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String, + vsys_id Int64 DEFAULT 1 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (vsys_id,common_recv_time,common_log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event ON CLUSTER ck_cluster +( + common_log_id UInt64, + common_recv_time Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String, + vsys_id Int64 +) +ENGINE = Distributed('ck_cluster', + 'tsg_galaxy_v3', + 'assessment_event_local', + rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event ON CLUSTER ck_query +( + common_log_id UInt64, + common_recv_time Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String, + vsys_id Int64 +) +ENGINE = Distributed('ck_cluster', + 'tsg_galaxy_v3', + 'assessment_event_local', + rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local ON CLUSTER ck_cluster( + log_id UInt64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64, + vsys_id Int64 DEFAULT 1 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(start_time)) +ORDER BY (vsys_id,destination_ip,start_time,log_id) +SETTINGS index_granularity = 8192; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event ON CLUSTER ck_cluster( + log_id UInt64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64, + vsys_id Int64 +) ENGINE = Distributed('ck_cluster', 'tsg_galaxy_v3', 'dos_event_local', rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event ON CLUSTER ck_query( + log_id UInt64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64, + vsys_id Int64 +) ENGINE = Distributed('ck_cluster', 'tsg_galaxy_v3', 'dos_event_local', rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_data_center,common_recv_time,common_log_id); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,gtpc_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,gtpc_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_data_center,common_device_group,common_recv_time,common_log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String + ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,interim_session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String + ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,interim_session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String + +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_stream_trace_id,common_data_center,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +)ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_data_center,common_recv_time,common_log_id); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) +ENGINE =MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_policy_id,common_recv_time,common_log_id); + + + +create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_event_local,rand()); + + + +create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_event_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_action,common_policy_id,common_recv_time,common_log_id); + + +create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String + ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String + ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_sub_action,common_policy_id,common_recv_time,common_log_id); + + + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_data_center,common_device_group,common_recv_time,common_log_id); + + +create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_local ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64, + radius_acct_authentic Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_vsys_id,common_data_center,common_recv_time,common_log_id); + + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64, + radius_acct_authentic Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_full_path String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64, + radius_acct_authentic Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_client_ip,common_server_ip,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_ip,common_client_ip,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (http_domain,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_http_domain_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_http_domain_local,rand()); + + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_onff_log_local ON CLUSTER ck_cluster( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64, + vsys_id Int64 DEFAULT 1 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(event_timestamp)) +ORDER BY (vsys_id,account,event_timestamp); + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_onff_log ON CLUSTER ck_cluster( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64, + vsys_id Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_onff_log ON CLUSTER ck_query( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64, + vsys_id Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); + + + +create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); +create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); +create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); +create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); +CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); +CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), +common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), +common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), +common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_local,rand()); + + + +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1; diff --git a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_OLD.sql similarity index 98% rename from Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql rename to Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_OLD.sql index 57490bf..d6519df 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_OLD.sql @@ -157,6 +157,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_local ON CLUSTER ck_cluster common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -264,6 +265,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_cluster( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -366,6 +368,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_query( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -469,6 +472,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record_local ON CLUSTER common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -725,6 +729,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cl common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -979,6 +984,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_qu common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -1233,6 +1239,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_ common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -1394,6 +1401,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluste common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -1551,6 +1559,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query( common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -1708,6 +1717,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local ON CLUSTER ck_cluster common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -1823,6 +1833,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -1935,6 +1946,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -2047,6 +2059,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event_local ON CLUST common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -2150,6 +2163,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_ common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -2250,6 +2264,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_ common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -2350,6 +2365,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local ON CLUSTER ck_clus common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -2607,6 +2623,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -2861,6 +2878,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query ( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -3116,6 +3134,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local ON CLUSTER ck_cluster common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -3287,6 +3306,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster ( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -3454,6 +3474,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query ( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -3621,6 +3642,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local ON CLUSTER ck_clus common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -3878,6 +3900,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster ( common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -4132,6 +4155,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query ( common_server_location LowCardinality(String), common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label LowCardinality(String), @@ -4386,6 +4410,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_local ON CLUSTER ck_clust common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -4511,6 +4536,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_cluster ( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -4632,6 +4658,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_query ( common_server_location String, common_server_asn String, common_app_id String, + common_app_full_path String, common_userdefine_app_name String, common_app_identify_info String, common_app_label String, @@ -4729,7 +4756,9 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip_local O common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_client_ip,common_server_ip,common_recv_time); @@ -4751,7 +4780,9 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); @@ -4772,7 +4803,9 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); @@ -4793,7 +4826,9 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip_local O common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_server_ip,common_client_ip,common_recv_time); @@ -4815,7 +4850,9 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); @@ -4836,7 +4873,9 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); @@ -4856,7 +4895,9 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLU common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (http_domain,common_recv_time); @@ -4878,7 +4919,9 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_http_domain_local,rand()); @@ -4898,14 +4941,16 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c common_client_port Int64, common_server_port Int64, common_app_label LowCardinality(String), - common_direction Nullable(Int64) + common_direction Nullable(Int64), + common_data_center String, + common_device_group String ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_http_domain_local,rand()); -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local;