diff --git a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql index 41347dd..dae55e8 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql @@ -4251,3 +4251,67 @@ SELECT FROM tsg_galaxy_v3.session_record_local WHERE empty(monitor_rule_list) = 0 ; +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record_local on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int32, + timestamp_us UInt64, + job_id String, + sled_ip String, + device_group String, + traffic_link_id Int32, + source_ip String, + source_port Int32, + destination_ip String, + destination_port Int32, + packet String, + packet_length Int32, + measurements String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,job_id,recv_time,timestamp_us); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int32, + timestamp_us UInt64, + job_id String, + sled_ip String, + device_group String, + traffic_link_id Int32, + source_ip String, + source_port Int32, + destination_ip String, + destination_port Int32, + packet String, + packet_length Int32, + measurements String +) +ENGINE = Distributed('ck_cluster', + 'tsg_galaxy_v3', + 'datapath_telemetry_record_local', + rand()); + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record on cluster ck_query ( + log_id UInt64, + recv_time Int64, + vsys_id Int32, + timestamp_us UInt64, + job_id String, + sled_ip String, + device_group String, + traffic_link_id Int32, + source_ip String, + source_port Int32, + destination_ip String, + destination_port Int32, + packet String, + packet_length Int32, + measurements String +) +ENGINE = Distributed('ck_cluster', + 'tsg_galaxy_v3', + 'datapath_telemetry_record_local', + rand()); diff --git a/Groot-Stream 最新全量配置模版/TSG/集中式/etl_datapath_telemetry_record_kafka_to_clickhouse b/Groot-Stream 最新全量配置模版/TSG/集中式/etl_datapath_telemetry_record_kafka_to_clickhouse new file mode 100644 index 0000000..14655a6 --- /dev/null +++ b/Groot-Stream 最新全量配置模版/TSG/集中式/etl_datapath_telemetry_record_kafka_to_clickhouse @@ -0,0 +1,71 @@ +sources: + kafka_source: + type: kafka + properties: + topic: DATAPATH-TELEMETRY-RECORD + kafka.bootstrap.servers: 192.168.44.11:9094,192.168.44.13:9094,192.168.44.14:9094,192.168.44.15:9094,192.168.44.16:9094 + kafka.session.timeout.ms: 60000 + kafka.max.poll.records: 3000 + kafka.max.partition.fetch.bytes: 31457280 + #kafka.security.protocol: SSL + #kafka.ssl.endpoint.identification.algorithm: "" + #kafka.ssl.keystore.location: /data/tsg/olap/flink/topology/data/keystore.jks + #kafka.ssl.keystore.password: 86cf0e2ffba3f541a6c6761313e5cc7e + #kafka.ssl.truststore.location: /data/tsg/olap/flink/topology/data/truststore.jks + #kafka.ssl.truststore.password: 86cf0e2ffba3f541a6c6761313e5cc7e + #kafka.ssl.key.password: 86cf0e2ffba3f541a6c6761313e5cc7e + kafka.security.protocol: SASL_PLAINTEXT + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.group.id: etl_datapath_telemetry_record_kafka_to_clickhouse-20230125 + kafka.auto.offset.reset: latest + format: msgpack + +processing_pipelines: + etl_processor: # [object] Processing Pipeline + type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + remove_fields: + output_fields: + functions: + - function: SNOWFLAKE_ID + lookup_fields: [ '' ] + output_fields: [ log_id ] + parameters: + data_center_id_num: 1 + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [ __timestamp ] + output_fields: [ recv_time ] + parameters: + precision: seconds + - function: BASE64_ENCODE_TO_STRING + output_fields: [ packet ] + parameters: + value_field: packet + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: 192.168.44.13:9001,192.168.44.14:9001,192.168.44.15:9001,192.168.44.16:9001 + table: tsg_galaxy_v3.datapath_telemetry_record_local + batch.size: 5000 + batch.interval: 30s + connection.user: e54c9568586180eede1506eecf3574e9 + connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e + + +application: + + env: # [object] Environment Variables + name: etl_datapath_telemetry_record_kafka_to_clickhouse # [string] Job Name + shade.identifier: aes + pipeline: + object-reuse: true # [boolean] Object Reuse, default is false + topology: + - name: kafka_source + downstream: [etl_processor] + - name: etl_processor + downstream: [clickhouse_sink] + - name: clickhouse_sink + + diff --git a/TSG发布版本更新记录/TSG-24.05/clickhouse/Clickhouse_TSG_建表语句.sql b/TSG发布版本更新记录/TSG-24.05/clickhouse/Clickhouse_TSG_建表语句.sql new file mode 100644 index 0000000..dae55e8 --- /dev/null +++ b/TSG发布版本更新记录/TSG-24.05/clickhouse/Clickhouse_TSG_建表语句.sql @@ -0,0 +1,4317 @@ +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query; + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + sessions Int64, + session_rate Int64, + packets Int64, + packet_rate Int64, + bytes Int64, + bit_rate Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,destination_ip,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + sessions Int64, + session_rate Int64, + packets Int64, + packet_rate Int64, + bytes Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_query ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + sessions Int64, + session_rate Int64, + packets Int64, + packet_rate Int64, + bytes Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,session_id,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +ip_protocol LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +ip_protocol LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +ip_protocol LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + +-- tsg_galaxy_v3.security_event_materialized_view +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.security_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + c2s_ttl Nullable(Int32), + s2c_ttl Nullable(Int32), + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_country String, + client_super_administrative_area String, + client_administrative_area String, + client_sub_administrative_area String, + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_country String, + server_super_administrative_area String, + server_administrative_area String, + server_sub_administrative_area String, + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app_transition String, + app LowCardinality(String), + app_debug_info String, + app_content String, + app_extra_info String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_starttls_flag Nullable(Int32), + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + encapsulation String, + dup_traffic_flag Nullable(Int32), + tunnel_id_list Array(Int64), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + c2s_ttl, + s2c_ttl, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + client_os_desc, + client_geolocation, + client_country, + client_super_administrative_area, + client_administrative_area, + client_sub_administrative_area, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_port, + server_os_desc, + server_geolocation, + server_country, + server_super_administrative_area, + server_administrative_area, + server_sub_administrative_area, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + app_debug_info, + app_content, + app_extra_info, + fqdn_category_list, + ip_protocol, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_starttls_flag, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_id_list, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(security_rule_list) = 0 +; + +-- tsg_galaxy_v3.monitor_event_materialized_view +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.monitor_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + c2s_ttl Nullable(Int32), + s2c_ttl Nullable(Int32), + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_country String, + client_super_administrative_area String, + client_administrative_area String, + client_sub_administrative_area String, + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_country String, + server_super_administrative_area String, + server_administrative_area String, + server_sub_administrative_area String, + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app_transition String, + app LowCardinality(String), + app_debug_info String, + app_content String, + app_extra_info String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_starttls_flag Nullable(Int32), + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + encapsulation String, + dup_traffic_flag Nullable(Int32), + tunnel_id_list Array(Int64), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + c2s_ttl, + s2c_ttl, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + client_os_desc, + client_geolocation, + client_country, + client_super_administrative_area, + client_administrative_area, + client_sub_administrative_area, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_port, + server_os_desc, + server_geolocation, + server_country, + server_super_administrative_area, + server_administrative_area, + server_sub_administrative_area, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + app_debug_info, + app_content, + app_extra_info, + fqdn_category_list, + ip_protocol, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_starttls_flag, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_id_list, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(monitor_rule_list) = 0 +; +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record_local on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int32, + timestamp_us UInt64, + job_id String, + sled_ip String, + device_group String, + traffic_link_id Int32, + source_ip String, + source_port Int32, + destination_ip String, + destination_port Int32, + packet String, + packet_length Int32, + measurements String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,job_id,recv_time,timestamp_us); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int32, + timestamp_us UInt64, + job_id String, + sled_ip String, + device_group String, + traffic_link_id Int32, + source_ip String, + source_port Int32, + destination_ip String, + destination_port Int32, + packet String, + packet_length Int32, + measurements String +) +ENGINE = Distributed('ck_cluster', + 'tsg_galaxy_v3', + 'datapath_telemetry_record_local', + rand()); + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record on cluster ck_query ( + log_id UInt64, + recv_time Int64, + vsys_id Int32, + timestamp_us UInt64, + job_id String, + sled_ip String, + device_group String, + traffic_link_id Int32, + source_ip String, + source_port Int32, + destination_ip String, + destination_port Int32, + packet String, + packet_length Int32, + measurements String +) +ENGINE = Distributed('ck_cluster', + 'tsg_galaxy_v3', + 'datapath_telemetry_record_local', + rand());