TSG-21394 clickhouse增加公共字段app_category

tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl.sql

@@ -163,6 +163,7 @@ server_fqdn String,
 server_domain String,
 app_transition String, 
 app LowCardinality(String),
+app_category String,
 app_debug_info String,
 app_content String,
 app_extra_info String,
@@ -408,6 +409,7 @@ server_fqdn String,
 server_domain String,
 app_transition String, 
 app LowCardinality(String),
+app_category String,
 app_debug_info String,
 app_content String,
 app_extra_info String,
@@ -651,6 +653,7 @@ server_fqdn String,
 server_domain String,
 app_transition String, 
 app LowCardinality(String),
+app_category String,
 app_debug_info String,
 app_content String,
 app_extra_info String,
@@ -895,6 +898,7 @@ server_fqdn String,
 server_domain String,
 app_transition String, 
 app LowCardinality(String),
+app_category String,
 app_debug_info String,
 app_content String,
 app_extra_info String,
@@ -1138,6 +1142,7 @@ server_fqdn String,
 server_domain String,
 app_transition String, 
 app LowCardinality(String),
+app_category String,
 app_debug_info String,
 app_content String,
 app_extra_info String,
@@ -1382,6 +1387,7 @@ server_fqdn String,
 server_domain String,
 app_transition String, 
 app LowCardinality(String),
+app_category String,
 app_debug_info String,
 app_content String,
 app_extra_info String,
@@ -1959,6 +1965,7 @@ server_fqdn String,
 server_domain String,
 app_transition String, 
 app LowCardinality(String),
+app_category String,
 app_debug_info String,
 app_content String,
 app_extra_info String,
@@ -2123,6 +2130,7 @@ server_fqdn String,
 server_domain String,
 app_transition String, 
 app LowCardinality(String),
+app_category String,
 app_debug_info String,
 app_content String,
 app_extra_info String,
@@ -2289,6 +2297,7 @@ TO tsg_galaxy_v3.security_event_local
     server_domain String,
     app_transition String, 
     app LowCardinality(String),
+    app_category String,
     app_debug_info String,
     app_content String,
     app_extra_info String,
@@ -2530,6 +2539,7 @@ SELECT
     server_domain,
     app_transition,
     app,
+    app_category,
     app_debug_info,
     app_content,
     app_extra_info,
@@ -2776,6 +2786,7 @@ TO tsg_galaxy_v3.monitor_event_local
     server_domain String,
     app_transition String, 
     app LowCardinality(String),
+    app_category String,
     app_debug_info String,
     app_content String,
     app_extra_info String,
@@ -3017,6 +3028,7 @@ SELECT
     server_domain,
     app_transition,
     app,
+    app_category,
     app_debug_info,
     app_content,
     app_extra_info,

tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl_check.sql

@@ -2,13 +2,13 @@ SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, asses
 FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
 FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
 FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

tsg_olap/upgrade/TSG-24.06/clickhouse/tsg_olap_clickhouse_ddl_check_24.06.sql

@@ -0,0 +1,21 @@
+SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
+FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
+FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
+FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
+FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT log_id, recv_time, vsys_id, timestamp_us, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
+FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+
+
+    

tsg_olap/upgrade/TSG-24.06/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.06.sql

@@ -0,0 +1,995 @@
+set distributed_ddl_task_timeout = 180;
+
+drop view if exists tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster;
+drop view if exists tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster;
+
+-- TSG-21394 clickhouse增加公共字段app_category
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS app_category String after app;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS app_category String after app;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS app_category String after app;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS app_category String after app;
+
+ALTER table tsg_galaxy_v3.monitor_event_local on cluster ck_cluster add column IF NOT EXISTS app_category String after app;
+ALTER table tsg_galaxy_v3.monitor_event on cluster ck_cluster add column IF NOT EXISTS app_category String after app;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS app_category String after app;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS app_category String after app;
+
+-- tsg_galaxy_v3.security_event_materialized_view
+CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster 
+TO tsg_galaxy_v3.security_event_local
+(
+    recv_time Int64,
+    log_id UInt64,
+    decoded_as String,
+    session_id UInt64,
+    start_timestamp_ms DateTime64(3),
+    end_timestamp_ms DateTime64(3),
+    duration_ms Int32,
+    tcp_handshake_latency_ms Nullable(Int32),
+    ingestion_time Int64,
+    processing_time Int64,
+    -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+    device_id String,
+    out_link_id Nullable(Int32),
+    in_link_id Nullable(Int32),
+    device_tag String,
+    data_center String,
+    device_group String,
+    sled_ip String,
+    address_type Int32,
+    direction String,
+    vsys_id Int32,
+    t_vsys_id Int32,
+    flags Int64,
+    flags_identify_info String,
+    c2s_ttl Nullable(Int32),
+    s2c_ttl Nullable(Int32),
+    security_rule_list Array(Int64),
+    security_action String,
+    monitor_rule_list Array(Int64),
+    shaping_rule_list Array(Int64),
+    proxy_rule_list Array(Int64),
+    statistics_rule_list Array(Int64),
+    sc_rule_list Array(Int64),
+    sc_rsp_raw Array(Int64),
+    sc_rsp_decrypted Array(Int64),
+    proxy_action String,
+    proxy_pinning_status Nullable(Int32),
+    proxy_intercept_status Nullable(Int32),
+    proxy_passthrough_reason String,
+    proxy_client_side_latency_ms Nullable(Int32),
+    proxy_server_side_latency_ms Nullable(Int32),
+    proxy_client_side_version String,
+    proxy_server_side_version String,
+    proxy_cert_verify Nullable(Int32),
+    proxy_intercept_error String,
+    monitor_mirrored_pkts Nullable(Int32),
+    monitor_mirrored_bytes Nullable(Int32),
+    client_ip String,
+    client_port Int32,
+    client_os_desc String,
+    client_geolocation LowCardinality(String),
+    client_country String,
+    client_super_administrative_area String,
+    client_administrative_area String,
+    client_sub_administrative_area String,
+    client_asn Nullable(Int64),
+    subscriber_id String,
+    imei String,
+    imsi String,
+    phone_number String,
+    apn String,
+    server_ip String,
+    server_port Int32,
+    server_os_desc String,
+    server_geolocation LowCardinality(String),
+    server_country String,
+    server_super_administrative_area String,
+    server_administrative_area String,
+    server_sub_administrative_area String,
+    server_asn Nullable(Int64),
+    server_fqdn String,
+    server_domain String,
+    app_transition String, 
+    app LowCardinality(String),
+    app_category String,
+    app_debug_info String,
+    app_content String,
+    app_extra_info String,
+    fqdn_category_list Array(Int64),
+    ip_protocol LowCardinality(String),
+    decoded_path LowCardinality(String),
+    dns_message_id Nullable(Int32),
+    dns_qr Nullable(Int32),
+    dns_opcode Nullable(Int32),
+    dns_aa Nullable(Int32),
+    dns_tc Nullable(Int32),
+    dns_rd Nullable(Int32),
+    dns_ra Nullable(Int32),
+    dns_rcode Nullable(Int32),
+    dns_qdcount Nullable(Int32),
+    dns_ancount Nullable(Int32),
+    dns_nscount Nullable(Int32),
+    dns_arcount Nullable(Int32),
+    dns_qname String,
+    dns_qtype Nullable(Int32),
+    dns_qclass Nullable(Int32),
+    dns_cname String,
+    dns_sub Nullable(Int32),
+    dns_rr String,
+    dns_response_latency_ms Nullable(Int32),
+    http_url String,
+    http_host String,
+    http_request_line String,
+    http_response_line String,
+    http_request_body String,
+    http_response_body String,
+    http_proxy_flag Nullable(Int32),
+    http_sequence Nullable(Int32),
+    http_cookie String,
+    http_referer String,
+    http_user_agent String,
+    http_request_content_length Nullable(Int64),
+    http_request_content_type String,
+    http_response_content_length Nullable(Int64),
+    http_response_content_type String,
+    http_set_cookie String,
+    http_version String,
+    http_status_code Nullable(Int32),
+    http_response_latency_ms Nullable(Int32),
+    http_session_duration_ms Nullable(Int32),
+    http_action_file_size Nullable(Int64),
+    ssl_version String,
+    ssl_sni String,
+    ssl_san String,
+    ssl_cn String,
+    ssl_handshake_latency_ms Nullable(Int32),
+    ssl_ja3_hash String,
+    ssl_ja3s_hash String,
+    ssl_cert_issuer String,
+    ssl_cert_subject String,
+    ssl_esni_flag Nullable(Int32),
+    ssl_ech_flag Nullable(Int32),
+    dtls_cookie String,
+    dtls_version  String,
+    dtls_sni String,
+    dtls_san String,
+    dtls_cn String,
+    dtls_handshake_latency_ms Nullable(Int32),
+    dtls_ja3_fingerprint String,
+    dtls_ja3_hash String,
+    dtls_cert_issuer String,
+    dtls_cert_subject String,
+    mail_protocol_type String,
+    mail_account String,
+    mail_from_cmd String,
+    mail_to_cmd String,
+    mail_from String,
+    mail_password String,
+    mail_to String,
+    mail_cc String,
+    mail_bcc String,
+    mail_subject String,
+    mail_subject_charset String,
+    mail_attachment_name String,
+    mail_attachment_name_charset String,
+    mail_starttls_flag Nullable(Int32),
+    mail_eml_file String,
+    ftp_account String,
+    ftp_url String,
+    ftp_link_type String,
+    quic_version String,
+    quic_sni String,
+    quic_user_agent String,
+    rdp_cookie String,
+    rdp_security_protocol String,
+    rdp_client_channels String,
+    rdp_keyboard_layout String,
+    rdp_client_version String,
+    rdp_client_name String,
+    rdp_client_product_id String,
+    rdp_desktop_width String,
+    rdp_desktop_height String,
+    rdp_requested_color_depth String,
+    rdp_certificate_type String,
+    rdp_certificate_count Nullable(Int32),
+    rdp_certificate_permanent Nullable(Int32),
+    rdp_encryption_level String,
+    rdp_encryption_method String,
+    ssh_version String,
+    ssh_auth_success String,
+    ssh_client_version String,
+    ssh_server_version String,
+    ssh_cipher_alg String,
+    ssh_mac_alg String,
+    ssh_compression_alg String,
+    ssh_kex_alg String,
+    ssh_host_key_alg String,
+    ssh_host_key String,
+    ssh_hassh String,
+    sip_call_id String,
+    sip_originator_description String,
+    sip_responder_description String,
+    sip_user_agent String,
+    sip_server String,
+    sip_originator_sdp_connect_ip String,
+    sip_originator_sdp_media_port Nullable(Int32),
+    sip_originator_sdp_media_type String,
+    sip_originator_sdp_content String,
+    sip_responder_sdp_connect_ip String,
+    sip_responder_sdp_media_port Nullable(Int32),
+    sip_responder_sdp_media_type String,
+    sip_responder_sdp_content String,
+    sip_duration_s Nullable(Int32),
+    sip_bye String,
+    rtp_payload_type_c2s Nullable(Int32),
+    rtp_payload_type_s2c Nullable(Int32),
+    rtp_pcap_path String,
+    rtp_originator_dir Nullable(Int32),
+    stratum_cryptocurrency String,
+    stratum_mining_pools String,
+    stratum_mining_program String,
+    stratum_mining_subscribe String,
+    sent_pkts Int64,
+    received_pkts Int64,
+    sent_bytes Int64,
+    received_bytes Int64,
+    tcp_c2s_ip_fragments Nullable(Int64),
+    tcp_s2c_ip_fragments Nullable(Int64),
+    tcp_c2s_lost_bytes Nullable(Int64),
+    tcp_s2c_lost_bytes Nullable(Int64),
+    tcp_c2s_o3_pkts Nullable(Int64),
+    tcp_s2c_o3_pkts Nullable(Int64),
+    tcp_c2s_rtx_pkts Nullable(Int64),
+    tcp_s2c_rtx_pkts Nullable(Int64),
+    tcp_c2s_rtx_bytes Nullable(Int64),
+    tcp_s2c_rtx_bytes Nullable(Int64),
+    tcp_rtt_ms Nullable(Int32),
+    tcp_client_isn Nullable(Int64),
+    tcp_server_isn Nullable(Int64),
+    packet_capture_file String,
+    in_src_mac String,
+    out_src_mac String,
+    in_dest_mac String,
+    out_dest_mac String,
+    encapsulation String,
+    dup_traffic_flag Nullable(Int32),
+    tunnel_id_list Array(Int64),
+    tunnel_endpoint_a_desc String,
+    tunnel_endpoint_b_desc String
+) 
+AS
+SELECT
+    recv_time,
+    log_id,
+    decoded_as,
+    session_id,
+    start_timestamp_ms,
+    end_timestamp_ms,
+    duration_ms,
+    tcp_handshake_latency_ms,
+    ingestion_time,
+    processing_time,
+    -- insert_time,
+    device_id,
+    out_link_id,
+    in_link_id,
+    device_tag,
+    data_center,
+    device_group,
+    sled_ip,
+    address_type,
+    direction,
+    vsys_id,
+    t_vsys_id,
+    flags,
+    flags_identify_info,
+    c2s_ttl,
+    s2c_ttl,
+    security_rule_list,
+    security_action,
+    monitor_rule_list,
+    shaping_rule_list,
+    proxy_rule_list,
+    statistics_rule_list,
+    sc_rule_list,
+    sc_rsp_raw,
+    sc_rsp_decrypted,
+    proxy_action,
+    proxy_pinning_status,
+    proxy_intercept_status,
+    proxy_passthrough_reason,
+    proxy_client_side_latency_ms,
+    proxy_server_side_latency_ms,
+    proxy_client_side_version,
+    proxy_server_side_version,
+    proxy_cert_verify,
+    proxy_intercept_error,
+    monitor_mirrored_pkts,
+    monitor_mirrored_bytes,
+    client_ip,
+    client_port,
+    client_os_desc,
+    client_geolocation,
+    client_country,
+    client_super_administrative_area,
+    client_administrative_area,
+    client_sub_administrative_area,
+    client_asn,
+    subscriber_id,
+    imei,
+    imsi,
+    phone_number,
+    apn,
+    server_ip,
+    server_port,
+    server_os_desc,
+    server_geolocation,
+    server_country,
+    server_super_administrative_area,
+    server_administrative_area,
+    server_sub_administrative_area,
+    server_asn,
+    server_fqdn,
+    server_domain,
+    app_transition,
+    app,
+    app_category,
+    app_debug_info,
+    app_content,
+    app_extra_info,
+    fqdn_category_list,
+    ip_protocol,
+    decoded_path,
+    dns_message_id,
+    dns_qr,
+    dns_opcode,
+    dns_aa,
+    dns_tc,
+    dns_rd,
+    dns_ra,
+    dns_rcode,
+    dns_qdcount,
+    dns_ancount,
+    dns_nscount,
+    dns_arcount,
+    dns_qname,
+    dns_qtype,
+    dns_qclass,
+    dns_cname,
+    dns_sub,
+    dns_rr,
+    dns_response_latency_ms,
+    http_url,
+    http_host,
+    http_request_line,
+    http_response_line,
+    http_request_body,
+    http_response_body,
+    http_proxy_flag,
+    http_sequence,
+    http_cookie,
+    http_referer,
+    http_user_agent,
+    http_request_content_length,
+    http_request_content_type,
+    http_response_content_length,
+    http_response_content_type,
+    http_set_cookie,
+    http_version,
+    http_status_code,
+    http_response_latency_ms,
+    http_session_duration_ms,
+    http_action_file_size,
+    ssl_version,
+    ssl_sni,
+    ssl_san,
+    ssl_cn,
+    ssl_handshake_latency_ms,
+    ssl_ja3_hash,
+    ssl_ja3s_hash,
+    ssl_cert_issuer,
+    ssl_cert_subject,
+    ssl_esni_flag,
+    ssl_ech_flag,
+    dtls_cookie,
+    dtls_version,
+    dtls_sni,
+    dtls_san,
+    dtls_cn,
+    dtls_handshake_latency_ms,
+    dtls_ja3_fingerprint,
+    dtls_ja3_hash,
+    dtls_cert_issuer,
+    dtls_cert_subject,
+    mail_protocol_type,
+    mail_account,
+    mail_from_cmd,
+    mail_to_cmd,
+    mail_from,
+    mail_password,
+    mail_to,
+    mail_cc,
+    mail_bcc,
+    mail_subject,
+    mail_subject_charset,
+    mail_attachment_name,
+    mail_attachment_name_charset,
+    mail_starttls_flag,
+    mail_eml_file,
+    ftp_account,
+    ftp_url,
+    ftp_link_type,
+    quic_version,
+    quic_sni,
+    quic_user_agent,
+    rdp_cookie,
+    rdp_security_protocol,
+    rdp_client_channels,
+    rdp_keyboard_layout,
+    rdp_client_version,
+    rdp_client_name,
+    rdp_client_product_id,
+    rdp_desktop_width,
+    rdp_desktop_height,
+    rdp_requested_color_depth,
+    rdp_certificate_type,
+    rdp_certificate_count,
+    rdp_certificate_permanent,
+    rdp_encryption_level,
+    rdp_encryption_method,
+    ssh_version,
+    ssh_auth_success,
+    ssh_client_version,
+    ssh_server_version,
+    ssh_cipher_alg,
+    ssh_mac_alg,
+    ssh_compression_alg,
+    ssh_kex_alg,
+    ssh_host_key_alg,
+    ssh_host_key,
+    ssh_hassh,
+    sip_call_id,
+    sip_originator_description,
+    sip_responder_description,
+    sip_user_agent,
+    sip_server,
+    sip_originator_sdp_connect_ip,
+    sip_originator_sdp_media_port,
+    sip_originator_sdp_media_type,
+    sip_originator_sdp_content,
+    sip_responder_sdp_connect_ip,
+    sip_responder_sdp_media_port,
+    sip_responder_sdp_media_type,
+    sip_responder_sdp_content,
+    sip_duration_s,
+    sip_bye,
+    rtp_payload_type_c2s,
+    rtp_payload_type_s2c,
+    rtp_pcap_path,
+    rtp_originator_dir,
+    stratum_cryptocurrency,
+    stratum_mining_pools,
+    stratum_mining_program,
+    stratum_mining_subscribe,
+    sent_pkts,
+    received_pkts,
+    sent_bytes,
+    received_bytes,
+    tcp_c2s_ip_fragments,
+    tcp_s2c_ip_fragments,
+    tcp_c2s_lost_bytes,
+    tcp_s2c_lost_bytes,
+    tcp_c2s_o3_pkts,
+    tcp_s2c_o3_pkts,
+    tcp_c2s_rtx_pkts,
+    tcp_s2c_rtx_pkts,
+    tcp_c2s_rtx_bytes,
+    tcp_s2c_rtx_bytes,
+    tcp_rtt_ms,
+    tcp_client_isn,
+    tcp_server_isn,
+    packet_capture_file,
+    in_src_mac,
+    out_src_mac,
+    in_dest_mac,
+    out_dest_mac,
+    encapsulation,
+    dup_traffic_flag,
+    tunnel_id_list,
+    tunnel_endpoint_a_desc,
+    tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.session_record_local
+WHERE empty(security_rule_list) = 0
+;
+
+-- tsg_galaxy_v3.monitor_event_materialized_view
+CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster 
+TO tsg_galaxy_v3.monitor_event_local
+(
+    recv_time Int64,
+    log_id UInt64,
+    decoded_as String,
+    session_id UInt64,
+    start_timestamp_ms DateTime64(3),
+    end_timestamp_ms DateTime64(3),
+    duration_ms Int32,
+    tcp_handshake_latency_ms Nullable(Int32),
+    ingestion_time Int64,
+    processing_time Int64,
+    -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+    device_id String,
+    out_link_id Nullable(Int32),
+    in_link_id Nullable(Int32),
+    device_tag String,
+    data_center String,
+    device_group String,
+    sled_ip String,
+    address_type Int32,
+    direction String,
+    vsys_id Int32,
+    t_vsys_id Int32,
+    flags Int64,
+    flags_identify_info String,
+    c2s_ttl Nullable(Int32),
+    s2c_ttl Nullable(Int32),
+    security_rule_list Array(Int64),
+    security_action String,
+    monitor_rule_list Array(Int64),
+    shaping_rule_list Array(Int64),
+    proxy_rule_list Array(Int64),
+    statistics_rule_list Array(Int64),
+    sc_rule_list Array(Int64),
+    sc_rsp_raw Array(Int64),
+    sc_rsp_decrypted Array(Int64),
+    proxy_action String,
+    proxy_pinning_status Nullable(Int32),
+    proxy_intercept_status Nullable(Int32),
+    proxy_passthrough_reason String,
+    proxy_client_side_latency_ms Nullable(Int32),
+    proxy_server_side_latency_ms Nullable(Int32),
+    proxy_client_side_version String,
+    proxy_server_side_version String,
+    proxy_cert_verify Nullable(Int32),
+    proxy_intercept_error String,
+    monitor_mirrored_pkts Nullable(Int32),
+    monitor_mirrored_bytes Nullable(Int32),
+    client_ip String,
+    client_port Int32,
+    client_os_desc String,
+    client_geolocation LowCardinality(String),
+    client_country String,
+    client_super_administrative_area String,
+    client_administrative_area String,
+    client_sub_administrative_area String,
+    client_asn Nullable(Int64),
+    subscriber_id String,
+    imei String,
+    imsi String,
+    phone_number String,
+    apn String,
+    server_ip String,
+    server_port Int32,
+    server_os_desc String,
+    server_geolocation LowCardinality(String),
+    server_country String,
+    server_super_administrative_area String,
+    server_administrative_area String,
+    server_sub_administrative_area String,
+    server_asn Nullable(Int64),
+    server_fqdn String,
+    server_domain String,
+    app_transition String, 
+    app LowCardinality(String),
+    app_category String,
+    app_debug_info String,
+    app_content String,
+    app_extra_info String,
+    fqdn_category_list Array(Int64),
+    ip_protocol LowCardinality(String),
+    decoded_path LowCardinality(String),
+    dns_message_id Nullable(Int32),
+    dns_qr Nullable(Int32),
+    dns_opcode Nullable(Int32),
+    dns_aa Nullable(Int32),
+    dns_tc Nullable(Int32),
+    dns_rd Nullable(Int32),
+    dns_ra Nullable(Int32),
+    dns_rcode Nullable(Int32),
+    dns_qdcount Nullable(Int32),
+    dns_ancount Nullable(Int32),
+    dns_nscount Nullable(Int32),
+    dns_arcount Nullable(Int32),
+    dns_qname String,
+    dns_qtype Nullable(Int32),
+    dns_qclass Nullable(Int32),
+    dns_cname String,
+    dns_sub Nullable(Int32),
+    dns_rr String,
+    dns_response_latency_ms Nullable(Int32),
+    http_url String,
+    http_host String,
+    http_request_line String,
+    http_response_line String,
+    http_request_body String,
+    http_response_body String,
+    http_proxy_flag Nullable(Int32),
+    http_sequence Nullable(Int32),
+    http_cookie String,
+    http_referer String,
+    http_user_agent String,
+    http_request_content_length Nullable(Int64),
+    http_request_content_type String,
+    http_response_content_length Nullable(Int64),
+    http_response_content_type String,
+    http_set_cookie String,
+    http_version String,
+    http_status_code Nullable(Int32),
+    http_response_latency_ms Nullable(Int32),
+    http_session_duration_ms Nullable(Int32),
+    http_action_file_size Nullable(Int64),
+    ssl_version String,
+    ssl_sni String,
+    ssl_san String,
+    ssl_cn String,
+    ssl_handshake_latency_ms Nullable(Int32),
+    ssl_ja3_hash String,
+    ssl_ja3s_hash String,
+    ssl_cert_issuer String,
+    ssl_cert_subject String,
+    ssl_esni_flag Nullable(Int32),
+    ssl_ech_flag Nullable(Int32),
+    dtls_cookie String,
+    dtls_version  String,
+    dtls_sni String,
+    dtls_san String,
+    dtls_cn String,
+    dtls_handshake_latency_ms Nullable(Int32),
+    dtls_ja3_fingerprint String,
+    dtls_ja3_hash String,
+    dtls_cert_issuer String,
+    dtls_cert_subject String,
+    mail_protocol_type String,
+    mail_account String,
+    mail_from_cmd String,
+    mail_to_cmd String,
+    mail_from String,
+    mail_password String,
+    mail_to String,
+    mail_cc String,
+    mail_bcc String,
+    mail_subject String,
+    mail_subject_charset String,
+    mail_attachment_name String,
+    mail_attachment_name_charset String,
+    mail_starttls_flag Nullable(Int32),
+    mail_eml_file String,
+    ftp_account String,
+    ftp_url String,
+    ftp_link_type String,
+    quic_version String,
+    quic_sni String,
+    quic_user_agent String,
+    rdp_cookie String,
+    rdp_security_protocol String,
+    rdp_client_channels String,
+    rdp_keyboard_layout String,
+    rdp_client_version String,
+    rdp_client_name String,
+    rdp_client_product_id String,
+    rdp_desktop_width String,
+    rdp_desktop_height String,
+    rdp_requested_color_depth String,
+    rdp_certificate_type String,
+    rdp_certificate_count Nullable(Int32),
+    rdp_certificate_permanent Nullable(Int32),
+    rdp_encryption_level String,
+    rdp_encryption_method String,
+    ssh_version String,
+    ssh_auth_success String,
+    ssh_client_version String,
+    ssh_server_version String,
+    ssh_cipher_alg String,
+    ssh_mac_alg String,
+    ssh_compression_alg String,
+    ssh_kex_alg String,
+    ssh_host_key_alg String,
+    ssh_host_key String,
+    ssh_hassh String,
+    sip_call_id String,
+    sip_originator_description String,
+    sip_responder_description String,
+    sip_user_agent String,
+    sip_server String,
+    sip_originator_sdp_connect_ip String,
+    sip_originator_sdp_media_port Nullable(Int32),
+    sip_originator_sdp_media_type String,
+    sip_originator_sdp_content String,
+    sip_responder_sdp_connect_ip String,
+    sip_responder_sdp_media_port Nullable(Int32),
+    sip_responder_sdp_media_type String,
+    sip_responder_sdp_content String,
+    sip_duration_s Nullable(Int32),
+    sip_bye String,
+    rtp_payload_type_c2s Nullable(Int32),
+    rtp_payload_type_s2c Nullable(Int32),
+    rtp_pcap_path String,
+    rtp_originator_dir Nullable(Int32),
+    stratum_cryptocurrency String,
+    stratum_mining_pools String,
+    stratum_mining_program String,
+    stratum_mining_subscribe String,
+    sent_pkts Int64,
+    received_pkts Int64,
+    sent_bytes Int64,
+    received_bytes Int64,
+    tcp_c2s_ip_fragments Nullable(Int64),
+    tcp_s2c_ip_fragments Nullable(Int64),
+    tcp_c2s_lost_bytes Nullable(Int64),
+    tcp_s2c_lost_bytes Nullable(Int64),
+    tcp_c2s_o3_pkts Nullable(Int64),
+    tcp_s2c_o3_pkts Nullable(Int64),
+    tcp_c2s_rtx_pkts Nullable(Int64),
+    tcp_s2c_rtx_pkts Nullable(Int64),
+    tcp_c2s_rtx_bytes Nullable(Int64),
+    tcp_s2c_rtx_bytes Nullable(Int64),
+    tcp_rtt_ms Nullable(Int32),
+    tcp_client_isn Nullable(Int64),
+    tcp_server_isn Nullable(Int64),
+    packet_capture_file String,
+    in_src_mac String,
+    out_src_mac String,
+    in_dest_mac String,
+    out_dest_mac String,
+    encapsulation String,
+    dup_traffic_flag Nullable(Int32),
+    tunnel_id_list Array(Int64),
+    tunnel_endpoint_a_desc String,
+    tunnel_endpoint_b_desc String
+) 
+AS
+SELECT
+    recv_time,
+    log_id,
+    decoded_as,
+    session_id,
+    start_timestamp_ms,
+    end_timestamp_ms,
+    duration_ms,
+    tcp_handshake_latency_ms,
+    ingestion_time,
+    processing_time,
+    -- insert_time,
+    device_id,
+    out_link_id,
+    in_link_id,
+    device_tag,
+    data_center,
+    device_group,
+    sled_ip,
+    address_type,
+    direction,
+    vsys_id,
+    t_vsys_id,
+    flags,
+    flags_identify_info,
+    c2s_ttl,
+    s2c_ttl,
+    security_rule_list,
+    security_action,
+    monitor_rule_list,
+    shaping_rule_list,
+    proxy_rule_list,
+    statistics_rule_list,
+    sc_rule_list,
+    sc_rsp_raw,
+    sc_rsp_decrypted,
+    proxy_action,
+    proxy_pinning_status,
+    proxy_intercept_status,
+    proxy_passthrough_reason,
+    proxy_client_side_latency_ms,
+    proxy_server_side_latency_ms,
+    proxy_client_side_version,
+    proxy_server_side_version,
+    proxy_cert_verify,
+    proxy_intercept_error,
+    monitor_mirrored_pkts,
+    monitor_mirrored_bytes,
+    client_ip,
+    client_port,
+    client_os_desc,
+    client_geolocation,
+    client_country,
+    client_super_administrative_area,
+    client_administrative_area,
+    client_sub_administrative_area,
+    client_asn,
+    subscriber_id,
+    imei,
+    imsi,
+    phone_number,
+    apn,
+    server_ip,
+    server_port,
+    server_os_desc,
+    server_geolocation,
+    server_country,
+    server_super_administrative_area,
+    server_administrative_area,
+    server_sub_administrative_area,
+    server_asn,
+    server_fqdn,
+    server_domain,
+    app_transition,
+    app,
+    app_category,
+    app_debug_info,
+    app_content,
+    app_extra_info,
+    fqdn_category_list,
+    ip_protocol,
+    decoded_path,
+    dns_message_id,
+    dns_qr,
+    dns_opcode,
+    dns_aa,
+    dns_tc,
+    dns_rd,
+    dns_ra,
+    dns_rcode,
+    dns_qdcount,
+    dns_ancount,
+    dns_nscount,
+    dns_arcount,
+    dns_qname,
+    dns_qtype,
+    dns_qclass,
+    dns_cname,
+    dns_sub,
+    dns_rr,
+    dns_response_latency_ms,
+    http_url,
+    http_host,
+    http_request_line,
+    http_response_line,
+    http_request_body,
+    http_response_body,
+    http_proxy_flag,
+    http_sequence,
+    http_cookie,
+    http_referer,
+    http_user_agent,
+    http_request_content_length,
+    http_request_content_type,
+    http_response_content_length,
+    http_response_content_type,
+    http_set_cookie,
+    http_version,
+    http_status_code,
+    http_response_latency_ms,
+    http_session_duration_ms,
+    http_action_file_size,
+    ssl_version,
+    ssl_sni,
+    ssl_san,
+    ssl_cn,
+    ssl_handshake_latency_ms,
+    ssl_ja3_hash,
+    ssl_ja3s_hash,
+    ssl_cert_issuer,
+    ssl_cert_subject,
+    ssl_esni_flag,
+    ssl_ech_flag,
+    dtls_cookie,
+    dtls_version,
+    dtls_sni,
+    dtls_san,
+    dtls_cn,
+    dtls_handshake_latency_ms,
+    dtls_ja3_fingerprint,
+    dtls_ja3_hash,
+    dtls_cert_issuer,
+    dtls_cert_subject,
+    mail_protocol_type,
+    mail_account,
+    mail_from_cmd,
+    mail_to_cmd,
+    mail_from,
+    mail_password,
+    mail_to,
+    mail_cc,
+    mail_bcc,
+    mail_subject,
+    mail_subject_charset,
+    mail_attachment_name,
+    mail_attachment_name_charset,
+    mail_starttls_flag,
+    mail_eml_file,
+    ftp_account,
+    ftp_url,
+    ftp_link_type,
+    quic_version,
+    quic_sni,
+    quic_user_agent,
+    rdp_cookie,
+    rdp_security_protocol,
+    rdp_client_channels,
+    rdp_keyboard_layout,
+    rdp_client_version,
+    rdp_client_name,
+    rdp_client_product_id,
+    rdp_desktop_width,
+    rdp_desktop_height,
+    rdp_requested_color_depth,
+    rdp_certificate_type,
+    rdp_certificate_count,
+    rdp_certificate_permanent,
+    rdp_encryption_level,
+    rdp_encryption_method,
+    ssh_version,
+    ssh_auth_success,
+    ssh_client_version,
+    ssh_server_version,
+    ssh_cipher_alg,
+    ssh_mac_alg,
+    ssh_compression_alg,
+    ssh_kex_alg,
+    ssh_host_key_alg,
+    ssh_host_key,
+    ssh_hassh,
+    sip_call_id,
+    sip_originator_description,
+    sip_responder_description,
+    sip_user_agent,
+    sip_server,
+    sip_originator_sdp_connect_ip,
+    sip_originator_sdp_media_port,
+    sip_originator_sdp_media_type,
+    sip_originator_sdp_content,
+    sip_responder_sdp_connect_ip,
+    sip_responder_sdp_media_port,
+    sip_responder_sdp_media_type,
+    sip_responder_sdp_content,
+    sip_duration_s,
+    sip_bye,
+    rtp_payload_type_c2s,
+    rtp_payload_type_s2c,
+    rtp_pcap_path,
+    rtp_originator_dir,
+    stratum_cryptocurrency,
+    stratum_mining_pools,
+    stratum_mining_program,
+    stratum_mining_subscribe,
+    sent_pkts,
+    received_pkts,
+    sent_bytes,
+    received_bytes,
+    tcp_c2s_ip_fragments,
+    tcp_s2c_ip_fragments,
+    tcp_c2s_lost_bytes,
+    tcp_s2c_lost_bytes,
+    tcp_c2s_o3_pkts,
+    tcp_s2c_o3_pkts,
+    tcp_c2s_rtx_pkts,
+    tcp_s2c_rtx_pkts,
+    tcp_c2s_rtx_bytes,
+    tcp_s2c_rtx_bytes,
+    tcp_rtt_ms,
+    tcp_client_isn,
+    tcp_server_isn,
+    packet_capture_file,
+    in_src_mac,
+    out_src_mac,
+    in_dest_mac,
+    out_dest_mac,
+    encapsulation,
+    dup_traffic_flag,
+    tunnel_id_list,
+    tunnel_endpoint_a_desc,
+    tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.session_record_local
+WHERE empty(monitor_rule_list) = 0
+;