From ed7fdd5f4614e999051a08f5d8468bcb0c9129bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=AF=E6=AF=85?= Date: Fri, 9 Dec 2022 10:05:06 +0000 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=96=B0=E6=95=B4=E7=90=86=20Clickhou?= =?UTF-8?q?se=5FTSG=5F=E5=BB=BA=E8=A1=A8=E8=AF=AD=E5=8F=A5.sql=20=E5=86=85?= =?UTF-8?q?=E7=9A=84=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Clickhouse_TSG_建表语句.sql | 3770 +++++++++-------- 1 file changed, 1901 insertions(+), 1869 deletions(-) diff --git a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql index 98796d8..2087f85 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql @@ -684,8 +684,7 @@ ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) PRIMARY KEY (common_log_id,common_data_center,common_recv_time) ORDER BY (common_log_id,common_data_center,common_recv_time,common_vsys_id); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_cluster( +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cluster( common_recv_time Int64, common_log_id UInt64, common_stream_trace_id UInt64, @@ -761,7 +760,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_ common_mirrored_bytes Int64, common_address_type Int64, common_schema_type LowCardinality(String), - common_vsys_id Int64 DEFAULT 1, + common_vsys_id Int64, common_t_vsys_id Int64, common_flags UInt64, common_device_tag String, @@ -805,6 +804,23 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_ http_response_latency_ms Int64, http_session_duration_ms Int64, http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, dns_message_id Int64, dns_qr Nullable(Int64), dns_opcode Nullable(Int64), @@ -824,6 +840,53 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_ dns_sub Int64, dns_rr String, dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, sip_call_id String, sip_originator_description String, sip_responder_description String, @@ -838,13 +901,41 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_ sip_responder_sdp_media_type String, sip_responder_sdp_content String, sip_duration_s Int64, - sip_bye String - -) -ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) -PRIMARY KEY (common_stream_trace_id,common_data_center,common_recv_time) -ORDER BY (common_stream_trace_id,common_data_center,common_recv_time,common_vsys_id); - + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String + ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,interim_session_record_local,rand()); CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_query( @@ -1100,7 +1191,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_qu rdp_cookie String ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,interim_session_record_local,rand()); -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cluster( + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_cluster( common_recv_time Int64, common_log_id UInt64, common_stream_trace_id UInt64, @@ -1176,261 +1268,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cl common_mirrored_bytes Int64, common_address_type Int64, common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - http_url String, - http_host String, - http_domain String, - http_request_line String, - http_response_line String, - http_request_header String, - http_response_header String, - http_request_content String, - http_response_content String, - http_request_body String, - http_response_body String, - http_request_body_key String, - http_response_body_key String, - http_proxy_flag Int64, - http_sequence Int64, - http_snapshot String, - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length String, - http_request_content_type String, - http_response_content_length String, - http_response_content_type String, - http_content_length String, - http_content_type String, - http_set_cookie String, - http_version String, - http_response_latency_ms Int64, - http_session_duration_ms Int64, - http_action_file_size Int64, - mail_protocol_type String, - mail_account String, - mail_to_cmd String, - mail_from_cmd String, - mail_from String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_content String, - mail_content_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_attachment_content String, - mail_eml_file String, - mail_snapshot String, - dns_message_id Int64, - dns_qr Nullable(Int64), - dns_opcode Nullable(Int64), - dns_aa Int64, - dns_tc Int64, - dns_rd Int64, - dns_ra Int64, - dns_rcode Int64, - dns_qdcount Int64, - dns_ancount Int64, - dns_nscount Int64, - dns_arcount Int64, - dns_qname String, - dns_qtype Int64, - dns_qclass Int64, - dns_cname String, - dns_sub Int64, - dns_rr String, - dns_response_latency_ms Int64, - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_pinningst Nullable(Int64), - ssl_intercept_state Nullable(Int64), - ssl_passthrough_reason String, - ssl_server_side_latency Nullable(Int64), - ssl_client_side_latency Nullable(Int64), - ssl_server_side_version String, - ssl_client_side_version String, - ssl_cert_verify Nullable(Int64), - ssl_error String, - ssl_con_latency_ms Int64, - ssl_ja3_fingerprint String, - ssl_ja3_hash String, - ssl_ja3s_fingerprint String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_con_latency_ms Int64, - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - quic_version String, - quic_sni String, - quic_user_agent String, - ftp_account String, - ftp_url String, - ftp_content String, - ftp_link_type String, - bgp_type Int64, - bgp_as_num String, - bgp_route String, - voip_calling_account String, - voip_called_account String, - voip_calling_number String, - voip_called_number String, - streaming_media_url String, - streaming_media_protocol String, - app_extra_info String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Int64, - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Int64, - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Int64, - sip_bye String, - rtp_payload_type_c2s Nullable(Int64), - rtp_payload_type_s2c Nullable(Int64), - rtp_pcap_path String, - rtp_originator_dir Int64, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - rdp_encryption_method String, - rdp_encryption_level String, - rdp_certificate_permanent Int64, - rdp_certificate_count Int64, - rdp_certificate_type String , - rdp_requested_color_depth String, - rdp_desktop_height String, - rdp_desktop_width String, - rdp_client_product_id String, - rdp_client_name String, - rdp_client_version String, - rdp_keyboard_layout String, - rdp_client_channels String, - rdp_security_protocol String, - rdp_cookie String - ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,interim_session_record_local,rand()); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location LowCardinality(String), - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location LowCardinality(String), - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label LowCardinality(String), - common_app_surrogate_id String, - common_l7_protocol LowCardinality(String), - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol LowCardinality(String), - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type LowCardinality(String), - common_vsys_id Int64, + common_vsys_id Int64 DEFAULT 1, common_t_vsys_id Int64, common_flags UInt64, common_device_tag String, @@ -1508,7 +1346,12 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query( sip_responder_sdp_content String, sip_duration_s Int64, sip_bye String -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +PRIMARY KEY (common_stream_trace_id,common_data_center,common_recv_time) +ORDER BY (common_stream_trace_id,common_data_center,common_recv_time,common_vsys_id); + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluster( common_recv_time Int64, @@ -1667,6 +1510,161 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluste ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local ON CLUSTER ck_cluster( @@ -1782,116 +1780,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local ON CLUSTER ck_cluster PRIMARY KEY (common_log_id,common_data_center,common_recv_time) ORDER BY (common_log_id,common_data_center,common_recv_time,common_vsys_id); -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location String, - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location String, - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label String, - common_app_surrogate_id String, - common_l7_protocol String, - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol String, - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type String, - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Int64, - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Int64, - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Int64, - sip_bye String, - rtp_payload_type_c2s Nullable(Int64), - rtp_payload_type_s2c Nullable(Int64), - rtp_pcap_path String, - rtp_originator_dir Int64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster( common_recv_time Int64, @@ -2004,6 +1893,119 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster( rtp_originator_dir Int64 ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event_local ON CLUSTER ck_cluster( common_recv_time Int64, common_log_id UInt64, @@ -2106,6 +2108,207 @@ PRIMARY KEY (common_log_id,common_policy_id,common_recv_time) ORDER BY (common_log_id,common_policy_id,common_recv_time,common_vsys_id); + +create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_event_local,rand()); + + + +create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_event_local,rand()); + + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local ON CLUSTER ck_cluster( common_recv_time Int64, common_log_id UInt64, @@ -2362,6 +2565,516 @@ ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) PRIMARY KEY (common_log_id,common_policy_id,common_recv_time) ORDER BY (common_log_id,common_policy_id,common_recv_time,common_vsys_id); + +create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String + ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + dns_response_latency_ms Int64, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_passthrough_reason String, + ssl_server_side_latency Nullable(Int64), + ssl_client_side_latency Nullable(Int64), + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_ja3s_fingerprint String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_con_latency_ms Int64, + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String + ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local ON CLUSTER ck_cluster( common_recv_time Int64, common_log_id UInt64, @@ -2532,6 +3245,341 @@ PRIMARY KEY (common_log_id,common_policy_id,common_recv_time) ORDER BY (common_log_id,common_policy_id,common_recv_time,common_vsys_id); + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length String, + http_request_content_type String, + http_response_content_length String, + http_response_content_type String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String, + rdp_encryption_method String, + rdp_encryption_level String, + rdp_certificate_permanent Int64, + rdp_certificate_count Int64, + rdp_certificate_type String , + rdp_requested_color_depth String, + rdp_desktop_height String, + rdp_desktop_width String, + rdp_client_product_id String, + rdp_client_name String, + rdp_client_version String, + rdp_keyboard_layout String, + rdp_client_channels String, + rdp_security_protocol String, + rdp_cookie String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local ON CLUSTER ck_cluster( common_recv_time Int64, common_log_id UInt64, @@ -2788,1235 +3836,6 @@ ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) PRIMARY KEY (common_log_id,common_data_center,common_recv_time) ORDER BY (common_log_id,common_data_center,common_recv_time,common_vsys_id); -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_local ON CLUSTER ck_cluster ( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location String, - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location String, - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label String, - common_app_surrogate_id String, - common_l7_protocol String, - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol String, - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type String, - common_vsys_id Int64 DEFAULT 1, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - radius_packet_type Int64, - radius_nas_ip String, - radius_framed_ip String, - radius_account String, - radius_session_timeout Int64, - radius_idle_timeout Int64, - radius_acct_status_type Int64, - radius_acct_terminate_cause Int64, - radius_event_timestamp Int64, - radius_nas_port Int64, - radius_service_type Int64, - radius_framed_protocol Int64, - radius_callback_number String, - radius_callback_id String, - radius_termination_action Nullable(Int64), - radius_called_station_id String, - radius_calling_station_id String, - radius_acct_delay_time Int64, - radius_acct_session_id String, - radius_acct_multi_session_id String, - radius_acct_input_octets Int64, - radius_acct_output_octets Int64, - radius_acct_input_packets Int64, - radius_acct_output_packets Int64, - radius_acct_session_time Int64, - radius_acct_link_count Int64, - radius_acct_interim_interval Int64, - radius_acct_authentic Int64 -) -ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) -PRIMARY KEY (common_log_id,common_data_center,common_recv_time) -ORDER BY (common_log_id,common_data_center,common_recv_time,common_vsys_id); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip_local ON CLUSTER ck_cluster( - common_log_id UInt64, - common_recv_time Int64, - common_server_ip String, - common_client_ip String, - common_sled_ip String, - common_entrance_id Int64, - common_subscriber_id String, - common_stream_trace_id UInt64, - http_domain String, - ssl_sni String, - common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_client_port Int64, - common_server_port Int64, - common_app_label LowCardinality(String), - common_direction Nullable(Int64) -) -ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) -ORDER BY (common_client_ip,common_server_ip,common_recv_time); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip_local ON CLUSTER ck_cluster( - common_log_id UInt64, - common_recv_time Int64, - common_server_ip String, - common_client_ip String, - common_sled_ip String, - common_entrance_id Int64, - common_subscriber_id String, - common_stream_trace_id UInt64, - http_domain String, - ssl_sni String, - common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_client_port Int64, - common_server_port Int64, - common_app_label LowCardinality(String), - common_direction Nullable(Int64) -) -ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) -ORDER BY (common_server_ip,common_client_ip,common_recv_time); - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLUSTER ck_cluster( - common_log_id UInt64, - common_recv_time Int64, - common_server_ip String, - common_client_ip String, - common_sled_ip String, - common_entrance_id Int64, - common_subscriber_id String, - common_stream_trace_id UInt64, - http_domain String, - ssl_sni String, - common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_client_port Int64, - common_server_port Int64, - common_app_label LowCardinality(String), - common_direction Nullable(Int64) -) -ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) -ORDER BY (http_domain,common_recv_time); - - -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; - - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_onff_log_local ON CLUSTER ck_cluster( - event_timestamp Int64, - account String, - framed_ip String, - acct_status_type Int64, - acct_session_id String, - acct_session_time Int64, - vsys_id Int64 DEFAULT 1 -) -ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(event_timestamp)) -PRIMARY KEY (account,event_timestamp) -ORDER BY (account,event_timestamp,vsys_id); - -create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_query ( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location String, - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location String, - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label String, - common_app_surrogate_id String, - common_l7_protocol String, - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol String, - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type String, - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - radius_packet_type Int64, - radius_nas_ip String, - radius_framed_ip String, - radius_account String, - radius_session_timeout Int64, - radius_idle_timeout Int64, - radius_acct_status_type Int64, - radius_acct_terminate_cause Int64, - radius_event_timestamp Int64, - radius_nas_port Int64, - radius_service_type Int64, - radius_framed_protocol Int64, - radius_callback_number String, - radius_callback_id String, - radius_termination_action Nullable(Int64), - radius_called_station_id String, - radius_calling_station_id String, - radius_acct_delay_time Int64, - radius_acct_session_id String, - radius_acct_multi_session_id String, - radius_acct_input_octets Int64, - radius_acct_output_octets Int64, - radius_acct_input_packets Int64, - radius_acct_output_packets Int64, - radius_acct_session_time Int64, - radius_acct_link_count Int64, - radius_acct_interim_interval Int64, - radius_acct_authentic Int64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_local,rand()); - -create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query ( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location LowCardinality(String), - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location LowCardinality(String), - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label LowCardinality(String), - common_app_surrogate_id String, - common_l7_protocol LowCardinality(String), - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol LowCardinality(String), - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - http_url String, - http_host String, - http_domain String, - http_request_line String, - http_response_line String, - http_request_header String, - http_response_header String, - http_request_content String, - http_response_content String, - http_request_body String, - http_response_body String, - http_request_body_key String, - http_response_body_key String, - http_proxy_flag Int64, - http_sequence Int64, - http_snapshot String, - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length String, - http_request_content_type String, - http_response_content_length String, - http_response_content_type String, - http_content_length String, - http_content_type String, - http_set_cookie String, - http_version String, - http_response_latency_ms Int64, - http_session_duration_ms Int64, - http_action_file_size Int64, - mail_protocol_type String, - mail_account String, - mail_to_cmd String, - mail_from_cmd String, - mail_from String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_content String, - mail_content_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_attachment_content String, - mail_eml_file String, - mail_snapshot String, - dns_message_id Int64, - dns_qr Nullable(Int64), - dns_opcode Nullable(Int64), - dns_aa Int64, - dns_tc Int64, - dns_rd Int64, - dns_ra Int64, - dns_rcode Int64, - dns_qdcount Int64, - dns_ancount Int64, - dns_nscount Int64, - dns_arcount Int64, - dns_qname String, - dns_qtype Int64, - dns_qclass Int64, - dns_cname String, - dns_sub Int64, - dns_rr String, - dns_response_latency_ms Int64, - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_pinningst Nullable(Int64), - ssl_intercept_state Nullable(Int64), - ssl_passthrough_reason String, - ssl_server_side_latency Nullable(Int64), - ssl_client_side_latency Nullable(Int64), - ssl_server_side_version String, - ssl_client_side_version String, - ssl_cert_verify Nullable(Int64), - ssl_error String, - ssl_con_latency_ms Int64, - ssl_ja3_fingerprint String, - ssl_ja3_hash String, - ssl_ja3s_fingerprint String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_con_latency_ms Int64, - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - quic_version String, - quic_sni String, - quic_user_agent String, - ftp_account String, - ftp_url String, - ftp_content String, - ftp_link_type String, - bgp_type Int64, - bgp_as_num String, - bgp_route String, - voip_calling_account String, - voip_called_account String, - voip_calling_number String, - voip_called_number String, - streaming_media_url String, - streaming_media_protocol String, - app_extra_info String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Int64, - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Int64, - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Int64, - sip_bye String, - rtp_payload_type_c2s Nullable(Int64), - rtp_payload_type_s2c Nullable(Int64), - rtp_pcap_path String, - rtp_originator_dir Int64, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - rdp_encryption_method String, - rdp_encryption_level String, - rdp_certificate_permanent Int64, - rdp_certificate_count Int64, - rdp_certificate_type String , - rdp_requested_color_depth String, - rdp_desktop_height String, - rdp_desktop_width String, - rdp_client_product_id String, - rdp_client_name String, - rdp_client_version String, - rdp_keyboard_layout String, - rdp_client_channels String, - rdp_security_protocol String, - rdp_cookie String -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); - -create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query ( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location String, - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location String, - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label String, - common_app_surrogate_id String, - common_l7_protocol String, - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol String, - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type String, - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - http_url String, - http_host String, - http_domain String, - http_request_line String, - http_response_line String, - http_request_header String, - http_response_header String, - http_request_content String, - http_response_content String, - http_request_body String, - http_response_body String, - http_request_body_key String, - http_response_body_key String, - http_proxy_flag Int64, - http_sequence Int64, - http_snapshot String, - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length String, - http_request_content_type String, - http_response_content_length String, - http_response_content_type String, - http_content_length String, - http_content_type String, - http_set_cookie String, - http_version String, - http_response_latency_ms Int64, - http_session_duration_ms Int64, - http_action_file_size Int64, - doh_url String, - doh_host String, - doh_request_line String, - doh_response_line String, - doh_cookie String, - doh_referer String, - doh_user_agent String, - doh_content_length String, - doh_content_type String, - doh_set_cookie String, - doh_version String, - doh_message_id Int64, - doh_qr Int64, - doh_opcode Int64, - doh_aa Int64, - doh_tc Int64, - doh_rd Int64, - doh_ra Int64, - doh_rcode Int64, - doh_qdcount Int64, - doh_ancount Int64, - doh_nscount Int64, - doh_arcount Int64, - doh_qname String, - doh_qtype Int64, - doh_qclass Int64, - doh_cname String, - doh_sub Int64, - doh_rr String, - rdp_encryption_method String, - rdp_encryption_level String, - rdp_certificate_permanent Int64, - rdp_certificate_count Int64, - rdp_certificate_type String , - rdp_requested_color_depth String, - rdp_desktop_height String, - rdp_desktop_width String, - rdp_client_product_id String, - rdp_client_name String, - rdp_client_version String, - rdp_keyboard_layout String, - rdp_client_channels String, - rdp_security_protocol String, - rdp_cookie String -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); - -create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query ( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location String, - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location String, - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label String, - common_app_surrogate_id String, - common_l7_protocol String, - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol String, - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type String, - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - http_url String, - http_host String, - http_domain String, - http_request_line String, - http_response_line String, - http_request_header String, - http_response_header String, - http_request_content String, - http_response_content String, - http_request_body String, - http_response_body String, - http_request_body_key String, - http_response_body_key String, - http_proxy_flag Int64, - http_sequence Int64, - http_snapshot String, - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length String, - http_request_content_type String, - http_response_content_length String, - http_response_content_type String, - http_content_length String, - http_content_type String, - http_set_cookie String, - http_version String, - http_response_latency_ms Int64, - http_session_duration_ms Int64, - http_action_file_size Int64, - mail_protocol_type String, - mail_account String, - mail_to_cmd String, - mail_from_cmd String, - mail_from String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_content String, - mail_content_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_attachment_content String, - mail_eml_file String, - mail_snapshot String, - dns_message_id Int64, - dns_qr Nullable(Int64), - dns_opcode Nullable(Int64), - dns_aa Int64, - dns_tc Int64, - dns_rd Int64, - dns_ra Int64, - dns_rcode Int64, - dns_qdcount Int64, - dns_ancount Int64, - dns_nscount Int64, - dns_arcount Int64, - dns_qname String, - dns_qtype Int64, - dns_qclass Int64, - dns_cname String, - dns_sub Int64, - dns_rr String, - dns_response_latency_ms Int64, - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_pinningst Nullable(Int64), - ssl_intercept_state Nullable(Int64), - ssl_passthrough_reason String, - ssl_server_side_latency Nullable(Int64), - ssl_client_side_latency Nullable(Int64), - ssl_server_side_version String, - ssl_client_side_version String, - ssl_cert_verify Nullable(Int64), - ssl_error String, - ssl_con_latency_ms Int64, - ssl_ja3_fingerprint String, - ssl_ja3_hash String, - ssl_ja3s_fingerprint String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_con_latency_ms Int64, - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - quic_version String, - quic_sni String, - quic_user_agent String, - ftp_account String, - ftp_url String, - ftp_content String, - ftp_link_type String, - bgp_type Int64, - bgp_as_num String, - bgp_route String, - voip_calling_account String, - voip_called_account String, - voip_calling_number String, - voip_called_number String, - streaming_media_url String, - streaming_media_protocol String, - app_extra_info String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Int64, - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Int64, - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Int64, - sip_bye String, - rtp_payload_type_c2s Nullable(Int64), - rtp_payload_type_s2c Nullable(Int64), - rtp_pcap_path String, - rtp_originator_dir Int64, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - rdp_encryption_method String, - rdp_encryption_level String, - rdp_certificate_permanent Int64, - rdp_certificate_count Int64, - rdp_certificate_type String , - rdp_requested_color_depth String, - rdp_desktop_height String, - rdp_desktop_width String, - rdp_client_product_id String, - rdp_client_name String, - rdp_client_version String, - rdp_keyboard_layout String, - rdp_client_channels String, - rdp_security_protocol String, - rdp_cookie String - ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); - -create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_query ( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location String, - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location String, - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label String, - common_app_surrogate_id String, - common_l7_protocol String, - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol String, - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type String, - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - nic_name String, - origin_source_mac String, - origin_dest_mac String, - packet_url String, - pcap_storage_task_id Int64, - pcap_storage_duration Int64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_event_local,rand()); - - -create table IF NOT EXISTS tsg_galaxy_v3.radius_onff_log ON CLUSTER ck_query( - event_timestamp Int64, - account String, - framed_ip String, - acct_status_type Int64, - acct_session_id String, - acct_session_time Int64, - vsys_id Int64 -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); - - -create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_cluster ( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location String, - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location String, - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label String, - common_app_surrogate_id String, - common_l7_protocol String, - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol String, - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type String, - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - radius_packet_type Int64, - radius_nas_ip String, - radius_framed_ip String, - radius_account String, - radius_session_timeout Int64, - radius_idle_timeout Int64, - radius_acct_status_type Int64, - radius_acct_terminate_cause Int64, - radius_event_timestamp Int64, - radius_nas_port Int64, - radius_service_type Int64, - radius_framed_protocol Int64, - radius_callback_number String, - radius_callback_id String, - radius_termination_action Nullable(Int64), - radius_called_station_id String, - radius_calling_station_id String, - radius_acct_delay_time Int64, - radius_acct_session_id String, - radius_acct_multi_session_id String, - radius_acct_input_octets Int64, - radius_acct_output_octets Int64, - radius_acct_input_packets Int64, - radius_acct_output_packets Int64, - radius_acct_session_time Int64, - radius_acct_link_count Int64, - radius_acct_interim_interval Int64, - radius_acct_authentic Int64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_local,rand()); create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster ( common_recv_time Int64, @@ -4271,173 +4090,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster ( rdp_cookie String ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); -create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster ( - common_recv_time Int64, - common_log_id UInt64, - common_stream_trace_id UInt64, - common_direction Nullable(Int64), - common_stream_dir Int64, - common_start_time Int64, - common_end_time Int64, - common_con_duration_ms Int64, - common_establish_latency_ms Int64, - common_processing_time Int64, - common_ingestion_time Int64, - common_entrance_id Int64, - common_device_id String, - common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), - common_isp String, - common_data_center String, - common_sled_ip String, - common_device_group String, - common_app_behavior String, - common_action Int64, - common_sub_action String, - common_policy_id Int64, - common_user_tags String, - common_user_region String, - common_client_ip String, - common_internal_ip String, - common_client_port Int64, - common_client_location String, - common_client_asn String, - common_subscriber_id String, - common_imei String, - common_imsi String, - common_phone_number String, - common_server_ip String, - common_external_ip String, - common_server_port Int64, - common_server_location String, - common_server_asn String, - common_app_id String, - common_userdefine_app_name String, - common_app_identify_info String, - common_app_label String, - common_app_surrogate_id String, - common_l7_protocol String, - common_protocol_label String, - common_service_category Array(Int64), - common_service Int64, - common_l4_protocol String, - common_sessions Int64, - common_c2s_pkt_num Int64, - common_s2c_pkt_num Int64, - common_c2s_pkt_diff Int64, - common_s2c_pkt_diff Int64, - common_c2s_byte_num Int64, - common_s2c_byte_num Int64, - common_c2s_byte_diff Int64, - common_s2c_byte_diff Int64, - common_c2s_ipfrag_num Int64, - common_s2c_ipfrag_num Int64, - common_c2s_tcp_lostlen Int64, - common_s2c_tcp_lostlen Int64, - common_c2s_tcp_unorder_num Int64, - common_s2c_tcp_unorder_num Int64, - common_c2s_pkt_retrans Int64, - common_s2c_pkt_retrans Int64, - common_c2s_byte_retrans Int64, - common_s2c_byte_retrans Int64, - common_first_ttl Int64, - common_tcp_client_isn Int64, - common_tcp_server_isn Int64, - common_mirrored_pkts Int64, - common_mirrored_bytes Int64, - common_address_type Int64, - common_schema_type String, - common_vsys_id Int64, - common_t_vsys_id Int64, - common_flags UInt64, - common_device_tag String, - common_encapsulation Int64, - common_tunnels String, - common_address_list String, - common_has_dup_traffic Int64, - common_stream_error String, - common_link_info_c2s String, - common_link_info_s2c String, - common_packet_capture_file String, - common_tunnel_endpoint_a_desc String, - common_tunnel_endpoint_b_desc String, - http_url String, - http_host String, - http_domain String, - http_request_line String, - http_response_line String, - http_request_header String, - http_response_header String, - http_request_content String, - http_response_content String, - http_request_body String, - http_response_body String, - http_request_body_key String, - http_response_body_key String, - http_proxy_flag Int64, - http_sequence Int64, - http_snapshot String, - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length String, - http_request_content_type String, - http_response_content_length String, - http_response_content_type String, - http_content_length String, - http_content_type String, - http_set_cookie String, - http_version String, - http_response_latency_ms Int64, - http_session_duration_ms Int64, - http_action_file_size Int64, - doh_url String, - doh_host String, - doh_request_line String, - doh_response_line String, - doh_cookie String, - doh_referer String, - doh_user_agent String, - doh_content_length String, - doh_content_type String, - doh_set_cookie String, - doh_version String, - doh_message_id Int64, - doh_qr Int64, - doh_opcode Int64, - doh_aa Int64, - doh_tc Int64, - doh_rd Int64, - doh_ra Int64, - doh_rcode Int64, - doh_qdcount Int64, - doh_ancount Int64, - doh_nscount Int64, - doh_arcount Int64, - doh_qname String, - doh_qtype Int64, - doh_qclass Int64, - doh_cname String, - doh_sub Int64, - doh_rr String, - rdp_encryption_method String, - rdp_encryption_level String, - rdp_certificate_permanent Int64, - rdp_certificate_count Int64, - rdp_certificate_type String , - rdp_requested_color_depth String, - rdp_desktop_height String, - rdp_desktop_width String, - rdp_client_product_id String, - rdp_client_name String, - rdp_client_version String, - rdp_keyboard_layout String, - rdp_client_channels String, - rdp_security_protocol String, - rdp_cookie String -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); -create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( +create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query ( common_recv_time Int64, common_log_id UInt64, common_stream_trace_id UInt64, @@ -4466,7 +4120,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( common_client_ip String, common_internal_ip String, common_client_port Int64, - common_client_location String, + common_client_location LowCardinality(String), common_client_asn String, common_subscriber_id String, common_imei String, @@ -4475,18 +4129,18 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( common_server_ip String, common_external_ip String, common_server_port Int64, - common_server_location String, + common_server_location LowCardinality(String), common_server_asn String, common_app_id String, common_userdefine_app_name String, common_app_identify_info String, - common_app_label String, + common_app_label LowCardinality(String), common_app_surrogate_id String, - common_l7_protocol String, + common_l7_protocol LowCardinality(String), common_protocol_label String, common_service_category Array(Int64), common_service Int64, - common_l4_protocol String, + common_l4_protocol LowCardinality(String), common_sessions Int64, common_c2s_pkt_num Int64, common_s2c_pkt_num Int64, @@ -4512,7 +4166,7 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( common_mirrored_pkts Int64, common_mirrored_bytes Int64, common_address_type Int64, - common_schema_type String, + common_schema_type LowCardinality(String), common_vsys_id Int64, common_t_vsys_id Int64, common_flags UInt64, @@ -4688,9 +4342,135 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( rdp_client_channels String, rdp_security_protocol String, rdp_cookie String - ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); -create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_cluster ( + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_local ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64 DEFAULT 1, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64, + radius_acct_authentic Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +PRIMARY KEY (common_log_id,common_data_center,common_recv_time) +ORDER BY (common_log_id,common_data_center,common_recv_time,common_vsys_id); + + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_cluster ( common_recv_time Int64, common_log_id UInt64, common_stream_trace_id UInt64, @@ -4780,36 +4560,181 @@ create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_ common_packet_capture_file String, common_tunnel_endpoint_a_desc String, common_tunnel_endpoint_b_desc String, - nic_name String, - origin_source_mac String, - origin_dest_mac String, - packet_url String, - pcap_storage_task_id Int64, - pcap_storage_duration Int64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_event_local,rand()); + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64, + radius_acct_authentic Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_local,rand()); -create table IF NOT EXISTS tsg_galaxy_v3.radius_onff_log ON CLUSTER ck_cluster( - event_timestamp Int64, - account String, - framed_ip String, - acct_status_type Int64, - acct_session_id String, - acct_session_time Int64, - vsys_id Int64 +create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_ingestion_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_device_group String, + common_app_behavior String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_identify_info String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_vsys_id Int64, + common_t_vsys_id Int64, + common_flags UInt64, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + common_packet_capture_file String, + common_tunnel_endpoint_a_desc String, + common_tunnel_endpoint_b_desc String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64, + radius_acct_authentic Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64) ) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_client_ip,common_server_ip,common_recv_time); - - create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); - create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); - create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); - create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); - CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); - CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); - - CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_cluster( common_log_id UInt64, common_recv_time Int64, @@ -4829,6 +4754,51 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS common_direction Nullable(Int64) ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64) +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64) +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_ip,common_client_ip,common_recv_time); + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_cluster( common_log_id UInt64, common_recv_time Int64, @@ -4849,43 +4819,6 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_cluster( - common_log_id UInt64, - common_recv_time Int64, - common_server_ip String, - common_client_ip String, - common_sled_ip String, - common_entrance_id Int64, - common_subscriber_id String, - common_stream_trace_id UInt64, - http_domain String, - ssl_sni String, - common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_client_port Int64, - common_server_port Int64, - common_app_label LowCardinality(String), - common_direction Nullable(Int64) -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_http_domain_local,rand()); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_query( - common_log_id UInt64, - common_recv_time Int64, - common_server_ip String, - common_client_ip String, - common_sled_ip String, - common_entrance_id Int64, - common_subscriber_id String, - common_stream_trace_id UInt64, - http_domain String, - ssl_sni String, - common_schema_type LowCardinality(String), - common_vsys_id Int64, - common_client_port Int64, - common_server_port Int64, - common_app_label LowCardinality(String), - common_direction Nullable(Int64) -) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_query( common_log_id UInt64, @@ -4907,6 +4840,48 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64) +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (http_domain,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64) +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_http_domain_local,rand()); + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_query( common_log_id UInt64, common_recv_time Int64, @@ -4928,6 +4903,59 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction FROM tsg_galaxy_v3.session_record_local; + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_onff_log_local ON CLUSTER ck_cluster( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64, + vsys_id Int64 DEFAULT 1 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(event_timestamp)) +PRIMARY KEY (account,event_timestamp) +ORDER BY (account,event_timestamp,vsys_id); + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_onff_log ON CLUSTER ck_cluster( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64, + vsys_id Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_onff_log ON CLUSTER ck_query( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64, + vsys_id Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); + + + +create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); +create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); +create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); +create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); +CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); +CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster( common_log_id UInt64, common_recv_time Int64, @@ -4956,33 +4984,6 @@ ad_generate_time Int64 ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); -create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query( -common_log_id UInt64, -common_recv_time Int64, -common_entrance_id Int64, -common_device_id String, -common_egress_link_id Nullable(Int64), -common_ingress_link_id Nullable(Int64), -common_policy_id Int64, -common_user_region String, -ad_method String, -ad_protocol String, -common_address_type Int64, -ad_target_ip String, -ad_target_port String, -ad_cc_target_url String, -ad_target_ip_location String, -ad_target_ip_asn String, -ad_claimed_src_ip_profile_id Int64, -ad_reflector_profile_id Int64, -ad_sent_pkt_num Int64, -ad_sent_byte_num Int64, -ad_cc_initiate_connection_num Int64, -ad_cc_established_connection_num Int64, -ad_cc_rejected_connection_num Int64, -ad_generate_time Int64 -) -ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_local,rand()); create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster( common_log_id UInt64, @@ -5012,6 +5013,37 @@ ad_generate_time Int64 ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), +common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_local,rand()); + + + alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1; alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1; alter table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1;