galaxy-deployment-tsg-olap-data-initialization/clickhouse/001_create_tsg_olap_clickhouse_table.sql

create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster;

CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster (
    vsys_id Int32,
    recv_time Int64,
    log_id UInt64,
    profile_id Int64,
    rule_id Int64,
    start_time Int64,
    end_time Int64,
    attack_type String,
    severity String,
    conditions String,
    destination_ip String,
    destination_country String,
    source_ip_list String,
    source_country_list String,
    sessions Int64,
    session_rate Int64,
    packets Int64,
    packet_rate Int64,
    bytes Int64,
    bit_rate Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id,destination_ip,recv_time,log_id);

CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster (
    vsys_id Int32,
    recv_time Int64,
    log_id UInt64,
    profile_id Int64,
    rule_id Int64,
    start_time Int64,
    end_time Int64,
    attack_type String,
    severity String,
    conditions String,
    destination_ip String,
    destination_country String,
    source_ip_list String,
    source_country_list String,
    sessions Int64,
    session_rate Int64,
    packets Int64,
    packet_rate Int64,
    bytes Int64,
    bit_rate Int64
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand());



CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster (
        log_id UInt64,
        recv_time Int64,
        vsys_id Int64,
        assessment_date Int64,
        lot_number String,
        file_name String,
        assessment_file        String,
        assessment_type        String,
        features String,
        size Int64,
        file_checksum_sha String
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id,recv_time,log_id);


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster (
        log_id UInt64,
        recv_time Int64,
        vsys_id Int64,
        assessment_date Int64,
        lot_number String,
        file_name String,
        assessment_file        String,
        assessment_type        String,
        features String,
        size Int64,
        file_checksum_sha String
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand());




CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
c2s_ttl Nullable(Int32),
s2c_ttl Nullable(Int32),
security_rule_list Array(Int64),
security_action String,
monitor_rule_list Array(Int64),
shaping_rule_list Array(Int64),
proxy_rule_list Array(Int64),
statistics_rule_list Array(Int64),
sc_rule_list Array(Int64),
sc_rsp_raw Array(Int64),
sc_rsp_decrypted Array(Int64),
proxy_action String,
proxy_pinning_status Nullable(Int32),
proxy_intercept_status Nullable(Int32),
proxy_passthrough_reason String,
proxy_client_side_latency_ms Nullable(Int32),
proxy_server_side_latency_ms Nullable(Int32),
proxy_client_side_version String,
proxy_server_side_version String,
proxy_cert_verify Nullable(Int32),
proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
app_category String,
app_debug_info String,
app_content String,
app_extra_info String,
fqdn_category_list Array(Int64),
ip_protocol LowCardinality(String),
decoded_path LowCardinality(String),
dns_message_id Nullable(Int32),
dns_qr Nullable(Int32),
dns_opcode Nullable(Int32),
dns_aa Nullable(Int32),
dns_tc Nullable(Int32),
dns_rd Nullable(Int32),
dns_ra Nullable(Int32),
dns_rcode Nullable(Int32),
dns_qdcount Nullable(Int32),
dns_ancount Nullable(Int32),
dns_nscount Nullable(Int32),
dns_arcount Nullable(Int32),
dns_qname String,
dns_qtype Nullable(Int32),
dns_qclass Nullable(Int32),
dns_cname String,
dns_sub Nullable(Int32),
dns_rr String,
dns_response_latency_ms Nullable(Int32),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_cn String,
ssl_handshake_latency_ms Nullable(Int32),
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
ssl_esni_flag Nullable(Int32),
ssl_ech_flag Nullable(Int32),
dtls_cookie String,
dtls_version  String,
dtls_sni String,
dtls_san String,
dtls_cn String,
dtls_handshake_latency_ms Nullable(Int32),
dtls_ja3_fingerprint String,
dtls_ja3_hash String,
dtls_cert_issuer String,
dtls_cert_subject String,
mail_protocol_type String,
mail_account String,
mail_from_cmd String,
mail_to_cmd String,
mail_from String,
mail_password String,
mail_to String,
mail_cc String,
mail_bcc String,
mail_subject String,
mail_subject_charset String,
mail_attachment_name String,
mail_attachment_name_charset String,
mail_starttls_flag Nullable(Int32),
mail_eml_file String,
ftp_account String,
ftp_url String,
ftp_link_type String,
quic_version String,
quic_sni String,
quic_user_agent String,
rdp_cookie String,
rdp_security_protocol String,
rdp_client_channels String,
rdp_keyboard_layout String,
rdp_client_version String,
rdp_client_name String,
rdp_client_product_id String,
rdp_desktop_width String,
rdp_desktop_height String,
rdp_requested_color_depth String,
rdp_certificate_type String,
rdp_certificate_count Nullable(Int32),
rdp_certificate_permanent Nullable(Int32),
rdp_encryption_level String,
rdp_encryption_method String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String,
rtp_payload_type_c2s Nullable(Int32),
rtp_payload_type_s2c Nullable(Int32),
rtp_pcap_path String,
rtp_originator_dir Nullable(Int32),
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
tcp_c2s_ip_fragments Nullable(Int64),
tcp_s2c_ip_fragments Nullable(Int64),
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_rtt_ms Nullable(Int32),
tcp_client_isn Nullable(Int64),
tcp_server_isn Nullable(Int64),
packet_capture_file String,
in_src_mac String,
out_src_mac String,
in_dest_mac String,
out_dest_mac String,
encapsulation String,
dup_traffic_flag Nullable(Int32),
tunnel_id_list Array(Int64),
tunnel_endpoint_a_desc String,
tunnel_endpoint_b_desc String
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time);


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64,
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
c2s_ttl Nullable(Int32),
s2c_ttl Nullable(Int32),
security_rule_list Array(Int64),
security_action String,
monitor_rule_list Array(Int64),
shaping_rule_list Array(Int64),
proxy_rule_list Array(Int64),
statistics_rule_list Array(Int64),
sc_rule_list Array(Int64),
sc_rsp_raw Array(Int64),
sc_rsp_decrypted Array(Int64),
proxy_action String,
proxy_pinning_status Nullable(Int32),
proxy_intercept_status Nullable(Int32),
proxy_passthrough_reason String,
proxy_client_side_latency_ms Nullable(Int32),
proxy_server_side_latency_ms Nullable(Int32),
proxy_client_side_version String,
proxy_server_side_version String,
proxy_cert_verify Nullable(Int32),
proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
app_category String,
app_debug_info String,
app_content String,
app_extra_info String,
fqdn_category_list Array(Int64),
ip_protocol LowCardinality(String),
decoded_path LowCardinality(String),
dns_message_id Nullable(Int32),
dns_qr Nullable(Int32),
dns_opcode Nullable(Int32),
dns_aa Nullable(Int32),
dns_tc Nullable(Int32),
dns_rd Nullable(Int32),
dns_ra Nullable(Int32),
dns_rcode Nullable(Int32),
dns_qdcount Nullable(Int32),
dns_ancount Nullable(Int32),
dns_nscount Nullable(Int32),
dns_arcount Nullable(Int32),
dns_qname String,
dns_qtype Nullable(Int32),
dns_qclass Nullable(Int32),
dns_cname String,
dns_sub Nullable(Int32),
dns_rr String,
dns_response_latency_ms Nullable(Int32),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_cn String,
ssl_handshake_latency_ms Nullable(Int32),
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
ssl_esni_flag Nullable(Int32),
ssl_ech_flag Nullable(Int32),
dtls_cookie String,
dtls_version  String,
dtls_sni String,
dtls_san String,
dtls_cn String,
dtls_handshake_latency_ms Nullable(Int32),
dtls_ja3_fingerprint String,
dtls_ja3_hash String,
dtls_cert_issuer String,
dtls_cert_subject String,
mail_protocol_type String,
mail_account String,
mail_from_cmd String,
mail_to_cmd String,
mail_from String,
mail_password String,
mail_to String,
mail_cc String,
mail_bcc String,
mail_subject String,
mail_subject_charset String,
mail_attachment_name String,
mail_attachment_name_charset String,
mail_starttls_flag Nullable(Int32),
mail_eml_file String,
ftp_account String,
ftp_url String,
ftp_link_type String,
quic_version String,
quic_sni String,
quic_user_agent String,
rdp_cookie String,
rdp_security_protocol String,
rdp_client_channels String,
rdp_keyboard_layout String,
rdp_client_version String,
rdp_client_name String,
rdp_client_product_id String,
rdp_desktop_width String,
rdp_desktop_height String,
rdp_requested_color_depth String,
rdp_certificate_type String,
rdp_certificate_count Nullable(Int32),
rdp_certificate_permanent Nullable(Int32),
rdp_encryption_level String,
rdp_encryption_method String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String,
rtp_payload_type_c2s Nullable(Int32),
rtp_payload_type_s2c Nullable(Int32),
rtp_pcap_path String,
rtp_originator_dir Nullable(Int32),
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
tcp_c2s_ip_fragments Nullable(Int64),
tcp_s2c_ip_fragments Nullable(Int64),
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_rtt_ms Nullable(Int32),
tcp_client_isn Nullable(Int64),
tcp_server_isn Nullable(Int64),
packet_capture_file String,
in_src_mac String,
out_src_mac String,
in_dest_mac String,
out_dest_mac String,
encapsulation String,
dup_traffic_flag Nullable(Int32),
tunnel_id_list Array(Int64),
tunnel_endpoint_a_desc String,
tunnel_endpoint_b_desc String
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand());


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
c2s_ttl Nullable(Int32),
s2c_ttl Nullable(Int32),
security_rule_list Array(Int64),
security_action String,
monitor_rule_list Array(Int64),
shaping_rule_list Array(Int64),
proxy_rule_list Array(Int64),
statistics_rule_list Array(Int64),
sc_rule_list Array(Int64),
sc_rsp_raw Array(Int64),
sc_rsp_decrypted Array(Int64),
proxy_action String,
proxy_pinning_status Nullable(Int32),
proxy_intercept_status Nullable(Int32),
proxy_passthrough_reason String,
proxy_client_side_latency_ms Nullable(Int32),
proxy_server_side_latency_ms Nullable(Int32),
proxy_client_side_version String,
proxy_server_side_version String,
proxy_cert_verify Nullable(Int32),
proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
app_category String,
app_debug_info String,
app_content String,
app_extra_info String,
fqdn_category_list Array(Int64),
ip_protocol LowCardinality(String),
decoded_path LowCardinality(String),
dns_message_id Nullable(Int32),
dns_qr Nullable(Int32),
dns_opcode Nullable(Int32),
dns_aa Nullable(Int32),
dns_tc Nullable(Int32),
dns_rd Nullable(Int32),
dns_ra Nullable(Int32),
dns_rcode Nullable(Int32),
dns_qdcount Nullable(Int32),
dns_ancount Nullable(Int32),
dns_nscount Nullable(Int32),
dns_arcount Nullable(Int32),
dns_qname String,
dns_qtype Nullable(Int32),
dns_qclass Nullable(Int32),
dns_cname String,
dns_sub Nullable(Int32),
dns_rr String,
dns_response_latency_ms Nullable(Int32),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_cn String,
ssl_handshake_latency_ms Nullable(Int32),
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
ssl_esni_flag Nullable(Int32),
ssl_ech_flag Nullable(Int32),
dtls_cookie String,
dtls_version  String,
dtls_sni String,
dtls_san String,
dtls_cn String,
dtls_handshake_latency_ms Nullable(Int32),
dtls_ja3_fingerprint String,
dtls_ja3_hash String,
dtls_cert_issuer String,
dtls_cert_subject String,
mail_protocol_type String,
mail_account String,
mail_from_cmd String,
mail_to_cmd String,
mail_from String,
mail_password String,
mail_to String,
mail_cc String,
mail_bcc String,
mail_subject String,
mail_subject_charset String,
mail_attachment_name String,
mail_attachment_name_charset String,
mail_starttls_flag Nullable(Int32),
mail_eml_file String,
ftp_account String,
ftp_url String,
ftp_link_type String,
quic_version String,
quic_sni String,
quic_user_agent String,
rdp_cookie String,
rdp_security_protocol String,
rdp_client_channels String,
rdp_keyboard_layout String,
rdp_client_version String,
rdp_client_name String,
rdp_client_product_id String,
rdp_desktop_width String,
rdp_desktop_height String,
rdp_requested_color_depth String,
rdp_certificate_type String,
rdp_certificate_count Nullable(Int32),
rdp_certificate_permanent Nullable(Int32),
rdp_encryption_level String,
rdp_encryption_method String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String,
rtp_payload_type_c2s Nullable(Int32),
rtp_payload_type_s2c Nullable(Int32),
rtp_pcap_path String,
rtp_originator_dir Nullable(Int32),
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
tcp_c2s_ip_fragments Nullable(Int64),
tcp_s2c_ip_fragments Nullable(Int64),
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_rtt_ms Nullable(Int32),
tcp_client_isn Nullable(Int64),
tcp_server_isn Nullable(Int64),
packet_capture_file String,
in_src_mac String,
out_src_mac String,
in_dest_mac String,
out_dest_mac String,
encapsulation String,
dup_traffic_flag Nullable(Int32),
tunnel_id_list Array(Int64),
tunnel_endpoint_a_desc String,
tunnel_endpoint_b_desc String
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time);

CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64 ,
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
c2s_ttl Nullable(Int32),
s2c_ttl Nullable(Int32),
security_rule_list Array(Int64),
security_action String,
monitor_rule_list Array(Int64),
shaping_rule_list Array(Int64),
proxy_rule_list Array(Int64),
statistics_rule_list Array(Int64),
sc_rule_list Array(Int64),
sc_rsp_raw Array(Int64),
sc_rsp_decrypted Array(Int64),
proxy_action String,
proxy_pinning_status Nullable(Int32),
proxy_intercept_status Nullable(Int32),
proxy_passthrough_reason String,
proxy_client_side_latency_ms Nullable(Int32),
proxy_server_side_latency_ms Nullable(Int32),
proxy_client_side_version String,
proxy_server_side_version String,
proxy_cert_verify Nullable(Int32),
proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
app_category String,
app_debug_info String,
app_content String,
app_extra_info String,
fqdn_category_list Array(Int64),
ip_protocol LowCardinality(String),
decoded_path LowCardinality(String),
dns_message_id Nullable(Int32),
dns_qr Nullable(Int32),
dns_opcode Nullable(Int32),
dns_aa Nullable(Int32),
dns_tc Nullable(Int32),
dns_rd Nullable(Int32),
dns_ra Nullable(Int32),
dns_rcode Nullable(Int32),
dns_qdcount Nullable(Int32),
dns_ancount Nullable(Int32),
dns_nscount Nullable(Int32),
dns_arcount Nullable(Int32),
dns_qname String,
dns_qtype Nullable(Int32),
dns_qclass Nullable(Int32),
dns_cname String,
dns_sub Nullable(Int32),
dns_rr String,
dns_response_latency_ms Nullable(Int32),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_cn String,
ssl_handshake_latency_ms Nullable(Int32),
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
ssl_esni_flag Nullable(Int32),
ssl_ech_flag Nullable(Int32),
dtls_cookie String,
dtls_version  String,
dtls_sni String,
dtls_san String,
dtls_cn String,
dtls_handshake_latency_ms Nullable(Int32),
dtls_ja3_fingerprint String,
dtls_ja3_hash String,
dtls_cert_issuer String,
dtls_cert_subject String,
mail_protocol_type String,
mail_account String,
mail_from_cmd String,
mail_to_cmd String,
mail_from String,
mail_password String,
mail_to String,
mail_cc String,
mail_bcc String,
mail_subject String,
mail_subject_charset String,
mail_attachment_name String,
mail_attachment_name_charset String,
mail_starttls_flag Nullable(Int32),
mail_eml_file String,
ftp_account String,
ftp_url String,
ftp_link_type String,
quic_version String,
quic_sni String,
quic_user_agent String,
rdp_cookie String,
rdp_security_protocol String,
rdp_client_channels String,
rdp_keyboard_layout String,
rdp_client_version String,
rdp_client_name String,
rdp_client_product_id String,
rdp_desktop_width String,
rdp_desktop_height String,
rdp_requested_color_depth String,
rdp_certificate_type String,
rdp_certificate_count Nullable(Int32),
rdp_certificate_permanent Nullable(Int32),
rdp_encryption_level String,
rdp_encryption_method String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String,
rtp_payload_type_c2s Nullable(Int32),
rtp_payload_type_s2c Nullable(Int32),
rtp_pcap_path String,
rtp_originator_dir Nullable(Int32),
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
tcp_c2s_ip_fragments Nullable(Int64),
tcp_s2c_ip_fragments Nullable(Int64),
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_rtt_ms Nullable(Int32),
tcp_client_isn Nullable(Int64),
tcp_server_isn Nullable(Int64),
packet_capture_file String,
in_src_mac String,
out_src_mac String,
in_dest_mac String,
out_dest_mac String,
encapsulation String,
dup_traffic_flag Nullable(Int32),
tunnel_id_list Array(Int64),
tunnel_endpoint_a_desc String,
tunnel_endpoint_b_desc String
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand());


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
c2s_ttl Nullable(Int32),
s2c_ttl Nullable(Int32),
security_rule_list Array(Int64),
security_action String,
monitor_rule_list Array(Int64),
shaping_rule_list Array(Int64),
proxy_rule_list Array(Int64),
statistics_rule_list Array(Int64),
sc_rule_list Array(Int64),
sc_rsp_raw Array(Int64),
sc_rsp_decrypted Array(Int64),
proxy_action String,
proxy_pinning_status Nullable(Int32),
proxy_intercept_status Nullable(Int32),
proxy_passthrough_reason String,
proxy_client_side_latency_ms Nullable(Int32),
proxy_server_side_latency_ms Nullable(Int32),
proxy_client_side_version String,
proxy_server_side_version String,
proxy_cert_verify Nullable(Int32),
proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
app_category String,
app_debug_info String,
app_content String,
app_extra_info String,
fqdn_category_list Array(Int64),
ip_protocol LowCardinality(String),
decoded_path LowCardinality(String),
dns_message_id Nullable(Int32),
dns_qr Nullable(Int32),
dns_opcode Nullable(Int32),
dns_aa Nullable(Int32),
dns_tc Nullable(Int32),
dns_rd Nullable(Int32),
dns_ra Nullable(Int32),
dns_rcode Nullable(Int32),
dns_qdcount Nullable(Int32),
dns_ancount Nullable(Int32),
dns_nscount Nullable(Int32),
dns_arcount Nullable(Int32),
dns_qname String,
dns_qtype Nullable(Int32),
dns_qclass Nullable(Int32),
dns_cname String,
dns_sub Nullable(Int32),
dns_rr String,
dns_response_latency_ms Nullable(Int32),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_cn String,
ssl_handshake_latency_ms Nullable(Int32),
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
ssl_esni_flag Nullable(Int32),
ssl_ech_flag Nullable(Int32),
dtls_cookie String,
dtls_version  String,
dtls_sni String,
dtls_san String,
dtls_cn String,
dtls_handshake_latency_ms Nullable(Int32),
dtls_ja3_fingerprint String,
dtls_ja3_hash String,
dtls_cert_issuer String,
dtls_cert_subject String,
mail_protocol_type String,
mail_account String,
mail_from_cmd String,
mail_to_cmd String,
mail_from String,
mail_password String,
mail_to String,
mail_cc String,
mail_bcc String,
mail_subject String,
mail_subject_charset String,
mail_attachment_name String,
mail_attachment_name_charset String,
mail_starttls_flag Nullable(Int32),
mail_eml_file String,
ftp_account String,
ftp_url String,
ftp_link_type String,
quic_version String,
quic_sni String,
quic_user_agent String,
rdp_cookie String,
rdp_security_protocol String,
rdp_client_channels String,
rdp_keyboard_layout String,
rdp_client_version String,
rdp_client_name String,
rdp_client_product_id String,
rdp_desktop_width String,
rdp_desktop_height String,
rdp_requested_color_depth String,
rdp_certificate_type String,
rdp_certificate_count Nullable(Int32),
rdp_certificate_permanent Nullable(Int32),
rdp_encryption_level String,
rdp_encryption_method String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String,
rtp_payload_type_c2s Nullable(Int32),
rtp_payload_type_s2c Nullable(Int32),
rtp_pcap_path String,
rtp_originator_dir Nullable(Int32),
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
tcp_c2s_ip_fragments Nullable(Int64),
tcp_s2c_ip_fragments Nullable(Int64),
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_rtt_ms Nullable(Int32),
tcp_client_isn Nullable(Int64),
tcp_server_isn Nullable(Int64),
packet_capture_file String,
in_src_mac String,
out_src_mac String,
in_dest_mac String,
out_dest_mac String,
encapsulation String,
dup_traffic_flag Nullable(Int32),
tunnel_id_list Array(Int64),
tunnel_endpoint_a_desc String,
tunnel_endpoint_b_desc String
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time);

CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64,
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
c2s_ttl Nullable(Int32),
s2c_ttl Nullable(Int32),
security_rule_list Array(Int64),
security_action String,
monitor_rule_list Array(Int64),
shaping_rule_list Array(Int64),
proxy_rule_list Array(Int64),
statistics_rule_list Array(Int64),
sc_rule_list Array(Int64),
sc_rsp_raw Array(Int64),
sc_rsp_decrypted Array(Int64),
proxy_action String,
proxy_pinning_status Nullable(Int32),
proxy_intercept_status Nullable(Int32),
proxy_passthrough_reason String,
proxy_client_side_latency_ms Nullable(Int32),
proxy_server_side_latency_ms Nullable(Int32),
proxy_client_side_version String,
proxy_server_side_version String,
proxy_cert_verify Nullable(Int32),
proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
app_category String,
app_debug_info String,
app_content String,
app_extra_info String,
fqdn_category_list Array(Int64),
ip_protocol LowCardinality(String),
decoded_path LowCardinality(String),
dns_message_id Nullable(Int32),
dns_qr Nullable(Int32),
dns_opcode Nullable(Int32),
dns_aa Nullable(Int32),
dns_tc Nullable(Int32),
dns_rd Nullable(Int32),
dns_ra Nullable(Int32),
dns_rcode Nullable(Int32),
dns_qdcount Nullable(Int32),
dns_ancount Nullable(Int32),
dns_nscount Nullable(Int32),
dns_arcount Nullable(Int32),
dns_qname String,
dns_qtype Nullable(Int32),
dns_qclass Nullable(Int32),
dns_cname String,
dns_sub Nullable(Int32),
dns_rr String,
dns_response_latency_ms Nullable(Int32),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_cn String,
ssl_handshake_latency_ms Nullable(Int32),
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
ssl_esni_flag Nullable(Int32),
ssl_ech_flag Nullable(Int32),
dtls_cookie String,
dtls_version  String,
dtls_sni String,
dtls_san String,
dtls_cn String,
dtls_handshake_latency_ms Nullable(Int32),
dtls_ja3_fingerprint String,
dtls_ja3_hash String,
dtls_cert_issuer String,
dtls_cert_subject String,
mail_protocol_type String,
mail_account String,
mail_from_cmd String,
mail_to_cmd String,
mail_from String,
mail_password String,
mail_to String,
mail_cc String,
mail_bcc String,
mail_subject String,
mail_subject_charset String,
mail_attachment_name String,
mail_attachment_name_charset String,
mail_starttls_flag Nullable(Int32),
mail_eml_file String,
ftp_account String,
ftp_url String,
ftp_link_type String,
quic_version String,
quic_sni String,
quic_user_agent String,
rdp_cookie String,
rdp_security_protocol String,
rdp_client_channels String,
rdp_keyboard_layout String,
rdp_client_version String,
rdp_client_name String,
rdp_client_product_id String,
rdp_desktop_width String,
rdp_desktop_height String,
rdp_requested_color_depth String,
rdp_certificate_type String,
rdp_certificate_count Nullable(Int32),
rdp_certificate_permanent Nullable(Int32),
rdp_encryption_level String,
rdp_encryption_method String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String,
rtp_payload_type_c2s Nullable(Int32),
rtp_payload_type_s2c Nullable(Int32),
rtp_pcap_path String,
rtp_originator_dir Nullable(Int32),
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
tcp_c2s_ip_fragments Nullable(Int64),
tcp_s2c_ip_fragments Nullable(Int64),
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_rtt_ms Nullable(Int32),
tcp_client_isn Nullable(Int64),
tcp_server_isn Nullable(Int64),
packet_capture_file String,
in_src_mac String,
out_src_mac String,
in_dest_mac String,
out_dest_mac String,
encapsulation String,
dup_traffic_flag Nullable(Int32),
tunnel_id_list Array(Int64),
tunnel_endpoint_a_desc String,
tunnel_endpoint_b_desc String
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand());


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
ingestion_time Int64,
processing_time Int64,
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
address_type Int32,
vsys_id Int32,
client_ip String,
client_port Int32,
server_ip String,
server_port Int32,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
dns_message_id Nullable(Int32),
dns_qr Nullable(Int32),
dns_opcode Nullable(Int32),
dns_aa Nullable(Int32),
dns_tc Nullable(Int32),
dns_rd Nullable(Int32),
dns_ra Nullable(Int32),
dns_rcode Nullable(Int32),
dns_qdcount Nullable(Int32),
dns_ancount Nullable(Int32),
dns_nscount Nullable(Int32),
dns_arcount Nullable(Int32),
dns_qname String,
dns_qtype Nullable(Int32),
dns_qclass Nullable(Int32),
dns_cname String,
dns_sub Nullable(Int32),
dns_rr String,
dns_response_latency_ms Nullable(Int32),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
mail_protocol_type String,
mail_account String,
mail_from_cmd String,
mail_to_cmd String,
mail_from String,
mail_password String,
mail_to String,
mail_cc String,
mail_bcc String,
mail_subject String,
mail_subject_charset String,
mail_attachment_name String,
mail_attachment_name_charset String,
mail_starttls_flag Nullable(Int32),
mail_eml_file String,
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id,session_id,recv_time);


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
ingestion_time Int64,
processing_time Int64,
insert_time Int64 ,
address_type Int32,
vsys_id Int32,
client_ip String,
client_port Int32,
server_ip String,
server_port Int32,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
dns_message_id Nullable(Int32),
dns_qr Nullable(Int32),
dns_opcode Nullable(Int32),
dns_aa Nullable(Int32),
dns_tc Nullable(Int32),
dns_rd Nullable(Int32),
dns_ra Nullable(Int32),
dns_rcode Nullable(Int32),
dns_qdcount Nullable(Int32),
dns_ancount Nullable(Int32),
dns_nscount Nullable(Int32),
dns_arcount Nullable(Int32),
dns_qname String,
dns_qtype Nullable(Int32),
dns_qclass Nullable(Int32),
dns_cname String,
dns_sub Nullable(Int32),
dns_rr String,
dns_response_latency_ms Nullable(Int32),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
mail_protocol_type String,
mail_account String,
mail_from_cmd String,
mail_to_cmd String,
mail_from String,
mail_password String,
mail_to String,
mail_cc String,
mail_bcc String,
mail_subject String,
mail_subject_charset String,
mail_attachment_name String,
mail_attachment_name_charset String,
mail_starttls_flag Nullable(Int32),
mail_eml_file String,
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand());



alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1;
alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1;


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
client_ip String,
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
server_ip String,
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
ip_protocol LowCardinality(String),
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String,
rtp_payload_type_c2s Nullable(Int32),
rtp_payload_type_s2c Nullable(Int32),
rtp_pcap_path String,
rtp_originator_dir Nullable(Int32),
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time);

CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64,
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
client_ip String,
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
server_ip String,
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
ip_protocol LowCardinality(String),
sip_call_id String,
sip_originator_description String,
sip_responder_description String,
sip_user_agent String,
sip_server String,
sip_originator_sdp_connect_ip String,
sip_originator_sdp_media_port Nullable(Int32),
sip_originator_sdp_media_type String,
sip_originator_sdp_content String,
sip_responder_sdp_connect_ip String,
sip_responder_sdp_media_port Nullable(Int32),
sip_responder_sdp_media_type String,
sip_responder_sdp_content String,
sip_duration_s Nullable(Int32),
sip_bye String,
sip_bye_reason String,
rtp_payload_type_c2s Nullable(Int32),
rtp_payload_type_s2c Nullable(Int32),
rtp_pcap_path String,
rtp_originator_dir Nullable(Int32),
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand());


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
c2s_ttl Nullable(Int32),
s2c_ttl Nullable(Int32),
security_rule_list Array(Int64),
security_action String,
monitor_rule_list Array(Int64),
shaping_rule_list Array(Int64),
proxy_rule_list Array(Int64),
statistics_rule_list Array(Int64),
sc_rule_list Array(Int64),
sc_rsp_raw Array(Int64),
sc_rsp_decrypted Array(Int64),
proxy_action String,
proxy_pinning_status Nullable(Int32),
proxy_intercept_status Nullable(Int32),
proxy_passthrough_reason String,
proxy_client_side_latency_ms Nullable(Int32),
proxy_server_side_latency_ms Nullable(Int32),
proxy_client_side_version String,
proxy_server_side_version String,
proxy_cert_verify Nullable(Int32),
proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
app_category String,
app_debug_info String,
app_content String,
app_extra_info String,
fqdn_category_list Array(Int64),
ip_protocol LowCardinality(String),
decoded_path LowCardinality(String),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
doh_url String,
doh_host String,
doh_request_line String,
doh_response_line String,
doh_cookie String,
doh_referer String,
doh_user_agent String,
doh_content_length String,
doh_content_type String,
doh_set_cookie String,
doh_version String,
doh_message_id Int64,
doh_qr Nullable(Int64),
doh_opcode Nullable(Int64),
doh_aa Nullable(Int64),
doh_tc Nullable(Int64),
doh_rd Nullable(Int64),
doh_ra Nullable(Int64),
doh_rcode Nullable(Int64),
doh_qdcount Nullable(Int64),
doh_ancount Nullable(Int64),
doh_nscount Nullable(Int64),
doh_arcount Nullable(Int64),
doh_qname String,
doh_qtype Nullable(Int64),
doh_qclass Nullable(Int64),
doh_cname String,
doh_sub Nullable(Int64),
doh_rr String,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
tcp_c2s_ip_fragments Nullable(Int64),
tcp_s2c_ip_fragments Nullable(Int64),
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_rtt_ms Nullable(Int32),
tcp_client_isn Nullable(Int64),
tcp_server_isn Nullable(Int64),
packet_capture_file String,
in_src_mac String,
out_src_mac String,
in_dest_mac String,
out_dest_mac String,
encapsulation String,
dup_traffic_flag Nullable(Int32),
tunnel_id_list Array(Int64),
tunnel_endpoint_a_desc String,
tunnel_endpoint_b_desc String
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time);


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster (
recv_time Int64,
log_id UInt64,
decoded_as String,
session_id UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int32,
tcp_handshake_latency_ms Nullable(Int32),
ingestion_time Int64,
processing_time Int64,
insert_time Int64,
device_id String,
out_link_id Nullable(Int32),
in_link_id Nullable(Int32),
device_tag String,
data_center String,
device_group String,
sled_ip String,
address_type Int32,
direction String,
vsys_id Int32,
t_vsys_id Int32,
flags Int64,
flags_identify_info String,
c2s_ttl Nullable(Int32),
s2c_ttl Nullable(Int32),
security_rule_list Array(Int64),
security_action String,
monitor_rule_list Array(Int64),
shaping_rule_list Array(Int64),
proxy_rule_list Array(Int64),
statistics_rule_list Array(Int64),
sc_rule_list Array(Int64),
sc_rsp_raw Array(Int64),
sc_rsp_decrypted Array(Int64),
proxy_action String,
proxy_pinning_status Nullable(Int32),
proxy_intercept_status Nullable(Int32),
proxy_passthrough_reason String,
proxy_client_side_latency_ms Nullable(Int32),
proxy_server_side_latency_ms Nullable(Int32),
proxy_client_side_version String,
proxy_server_side_version String,
proxy_cert_verify Nullable(Int32),
proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
client_country String,
client_super_administrative_area String,
client_administrative_area String,
client_sub_administrative_area String,
client_asn Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
server_country String,
server_super_administrative_area String,
server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
app_category String,
app_debug_info String,
app_content String,
app_extra_info String,
fqdn_category_list Array(Int64),
ip_protocol LowCardinality(String),
decoded_path LowCardinality(String),
http_url String,
http_host String,
http_request_line String,
http_response_line String,
http_request_body String,
http_response_body String,
http_proxy_flag Nullable(Int32),
http_sequence Nullable(Int32),
http_cookie String,
http_referer String,
http_user_agent String,
http_request_content_length Nullable(Int64),
http_request_content_type String,
http_response_content_length Nullable(Int64),
http_response_content_type String,
http_set_cookie String,
http_version String,
http_status_code Nullable(Int32),
http_response_latency_ms Nullable(Int32),
http_session_duration_ms Nullable(Int32),
http_action_file_size Nullable(Int64),
doh_url String,
doh_host String,
doh_request_line String,
doh_response_line String,
doh_cookie String,
doh_referer String,
doh_user_agent String,
doh_content_length String,
doh_content_type String,
doh_set_cookie String,
doh_version String,
doh_message_id Int64,
doh_qr Nullable(Int64),
doh_opcode Nullable(Int64),
doh_aa Nullable(Int64),
doh_tc Nullable(Int64),
doh_rd Nullable(Int64),
doh_ra Nullable(Int64),
doh_rcode Nullable(Int64),
doh_qdcount Nullable(Int64),
doh_ancount Nullable(Int64),
doh_nscount Nullable(Int64),
doh_arcount Nullable(Int64),
doh_qname String,
doh_qtype Nullable(Int64),
doh_qclass Nullable(Int64),
doh_cname String,
doh_sub Nullable(Int64),
doh_rr String,
sent_pkts Int64,
received_pkts Int64,
sent_bytes Int64,
received_bytes Int64,
tcp_c2s_ip_fragments Nullable(Int64),
tcp_s2c_ip_fragments Nullable(Int64),
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_rtt_ms Nullable(Int32),
tcp_client_isn Nullable(Int64),
tcp_server_isn Nullable(Int64),
packet_capture_file String,
in_src_mac String,
out_src_mac String,
in_dest_mac String,
out_dest_mac String,
encapsulation String,
dup_traffic_flag Nullable(Int32),
tunnel_id_list Array(Int64),
tunnel_endpoint_a_desc String,
tunnel_endpoint_b_desc String
)
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand());


-- tsg_galaxy_v3.security_event_materialized_view
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster 
TO tsg_galaxy_v3.security_event_local
(
    recv_time Int64,
    log_id UInt64,
    decoded_as String,
    session_id UInt64,
    start_timestamp_ms DateTime64(3),
    end_timestamp_ms DateTime64(3),
    duration_ms Int32,
    tcp_handshake_latency_ms Nullable(Int32),
    ingestion_time Int64,
    processing_time Int64,
    -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
    device_id String,
    out_link_id Nullable(Int32),
    in_link_id Nullable(Int32),
    device_tag String,
    data_center String,
    device_group String,
    sled_ip String,
    address_type Int32,
    direction String,
    vsys_id Int32,
    t_vsys_id Int32,
    flags Int64,
    flags_identify_info String,
    c2s_ttl Nullable(Int32),
    s2c_ttl Nullable(Int32),
    security_rule_list Array(Int64),
    security_action String,
    monitor_rule_list Array(Int64),
    shaping_rule_list Array(Int64),
    proxy_rule_list Array(Int64),
    statistics_rule_list Array(Int64),
    sc_rule_list Array(Int64),
    sc_rsp_raw Array(Int64),
    sc_rsp_decrypted Array(Int64),
    proxy_action String,
    proxy_pinning_status Nullable(Int32),
    proxy_intercept_status Nullable(Int32),
    proxy_passthrough_reason String,
    proxy_client_side_latency_ms Nullable(Int32),
    proxy_server_side_latency_ms Nullable(Int32),
    proxy_client_side_version String,
    proxy_server_side_version String,
    proxy_cert_verify Nullable(Int32),
    proxy_intercept_error String,
    monitor_mirrored_pkts Nullable(Int32),
    monitor_mirrored_bytes Nullable(Int32),
    client_ip String,
    client_ip_tags Array(String),
    client_port Int32,
    client_os_desc String,
    client_geolocation LowCardinality(String),
    client_country String,
    client_super_administrative_area String,
    client_administrative_area String,
    client_sub_administrative_area String,
    client_asn Nullable(Int64),
    subscriber_id String,
    imei String,
    imsi String,
    phone_number String,
    apn String,
    server_ip String,
    server_ip_tags Array(String),
    server_port Int32,
    server_os_desc String,
    server_geolocation LowCardinality(String),
    server_country String,
    server_super_administrative_area String,
    server_administrative_area String,
    server_sub_administrative_area String,
    server_asn Nullable(Int64),
    server_fqdn String,
    server_fqdn_tags Array(String),
    server_domain String,
    app_transition String, 
    app LowCardinality(String),
    app_category String,
    app_debug_info String,
    app_content String,
    app_extra_info String,
    fqdn_category_list Array(Int64),
    ip_protocol LowCardinality(String),
    decoded_path LowCardinality(String),
    dns_message_id Nullable(Int32),
    dns_qr Nullable(Int32),
    dns_opcode Nullable(Int32),
    dns_aa Nullable(Int32),
    dns_tc Nullable(Int32),
    dns_rd Nullable(Int32),
    dns_ra Nullable(Int32),
    dns_rcode Nullable(Int32),
    dns_qdcount Nullable(Int32),
    dns_ancount Nullable(Int32),
    dns_nscount Nullable(Int32),
    dns_arcount Nullable(Int32),
    dns_qname String,
    dns_qtype Nullable(Int32),
    dns_qclass Nullable(Int32),
    dns_cname String,
    dns_sub Nullable(Int32),
    dns_rr String,
    dns_response_latency_ms Nullable(Int32),
    http_url String,
    http_host String,
    http_request_line String,
    http_response_line String,
    http_request_body String,
    http_response_body String,
    http_proxy_flag Nullable(Int32),
    http_sequence Nullable(Int32),
    http_cookie String,
    http_referer String,
    http_user_agent String,
    http_request_content_length Nullable(Int64),
    http_request_content_type String,
    http_response_content_length Nullable(Int64),
    http_response_content_type String,
    http_set_cookie String,
    http_version String,
    http_status_code Nullable(Int32),
    http_response_latency_ms Nullable(Int32),
    http_session_duration_ms Nullable(Int32),
    http_action_file_size Nullable(Int64),
    ssl_version String,
    ssl_sni String,
    ssl_san String,
    ssl_cn String,
    ssl_handshake_latency_ms Nullable(Int32),
    ssl_ja3_hash String,
    ssl_ja3s_hash String,
    ssl_cert_issuer String,
    ssl_cert_subject String,
    ssl_esni_flag Nullable(Int32),
    ssl_ech_flag Nullable(Int32),
    dtls_cookie String,
    dtls_version  String,
    dtls_sni String,
    dtls_san String,
    dtls_cn String,
    dtls_handshake_latency_ms Nullable(Int32),
    dtls_ja3_fingerprint String,
    dtls_ja3_hash String,
    dtls_cert_issuer String,
    dtls_cert_subject String,
    mail_protocol_type String,
    mail_account String,
    mail_from_cmd String,
    mail_to_cmd String,
    mail_from String,
    mail_password String,
    mail_to String,
    mail_cc String,
    mail_bcc String,
    mail_subject String,
    mail_subject_charset String,
    mail_attachment_name String,
    mail_attachment_name_charset String,
    mail_starttls_flag Nullable(Int32),
    mail_eml_file String,
    ftp_account String,
    ftp_url String,
    ftp_link_type String,
    quic_version String,
    quic_sni String,
    quic_user_agent String,
    rdp_cookie String,
    rdp_security_protocol String,
    rdp_client_channels String,
    rdp_keyboard_layout String,
    rdp_client_version String,
    rdp_client_name String,
    rdp_client_product_id String,
    rdp_desktop_width String,
    rdp_desktop_height String,
    rdp_requested_color_depth String,
    rdp_certificate_type String,
    rdp_certificate_count Nullable(Int32),
    rdp_certificate_permanent Nullable(Int32),
    rdp_encryption_level String,
    rdp_encryption_method String,
    ssh_version String,
    ssh_auth_success String,
    ssh_client_version String,
    ssh_server_version String,
    ssh_cipher_alg String,
    ssh_mac_alg String,
    ssh_compression_alg String,
    ssh_kex_alg String,
    ssh_host_key_alg String,
    ssh_host_key String,
    ssh_hassh String,
    sip_call_id String,
    sip_originator_description String,
    sip_responder_description String,
    sip_user_agent String,
    sip_server String,
    sip_originator_sdp_connect_ip String,
    sip_originator_sdp_media_port Nullable(Int32),
    sip_originator_sdp_media_type String,
    sip_originator_sdp_content String,
    sip_responder_sdp_connect_ip String,
    sip_responder_sdp_media_port Nullable(Int32),
    sip_responder_sdp_media_type String,
    sip_responder_sdp_content String,
    sip_duration_s Nullable(Int32),
    sip_bye String,
    sip_bye_reason String,
    rtp_payload_type_c2s Nullable(Int32),
    rtp_payload_type_s2c Nullable(Int32),
    rtp_pcap_path String,
    rtp_originator_dir Nullable(Int32),
    stratum_cryptocurrency String,
    stratum_mining_pools String,
    stratum_mining_program String,
    stratum_mining_subscribe String,
    sent_pkts Int64,
    received_pkts Int64,
    sent_bytes Int64,
    received_bytes Int64,
    tcp_c2s_ip_fragments Nullable(Int64),
    tcp_s2c_ip_fragments Nullable(Int64),
    tcp_c2s_lost_bytes Nullable(Int64),
    tcp_s2c_lost_bytes Nullable(Int64),
    tcp_c2s_o3_pkts Nullable(Int64),
    tcp_s2c_o3_pkts Nullable(Int64),
    tcp_c2s_rtx_pkts Nullable(Int64),
    tcp_s2c_rtx_pkts Nullable(Int64),
    tcp_c2s_rtx_bytes Nullable(Int64),
    tcp_s2c_rtx_bytes Nullable(Int64),
    tcp_rtt_ms Nullable(Int32),
    tcp_client_isn Nullable(Int64),
    tcp_server_isn Nullable(Int64),
    packet_capture_file String,
    in_src_mac String,
    out_src_mac String,
    in_dest_mac String,
    out_dest_mac String,
    encapsulation String,
    dup_traffic_flag Nullable(Int32),
    tunnel_id_list Array(Int64),
    tunnel_endpoint_a_desc String,
    tunnel_endpoint_b_desc String
) 
AS
SELECT
    recv_time,
    log_id,
    decoded_as,
    session_id,
    start_timestamp_ms,
    end_timestamp_ms,
    duration_ms,
    tcp_handshake_latency_ms,
    ingestion_time,
    processing_time,
    -- insert_time,
    device_id,
    out_link_id,
    in_link_id,
    device_tag,
    data_center,
    device_group,
    sled_ip,
    address_type,
    direction,
    vsys_id,
    t_vsys_id,
    flags,
    flags_identify_info,
    c2s_ttl,
    s2c_ttl,
    security_rule_list,
    security_action,
    monitor_rule_list,
    shaping_rule_list,
    proxy_rule_list,
    statistics_rule_list,
    sc_rule_list,
    sc_rsp_raw,
    sc_rsp_decrypted,
    proxy_action,
    proxy_pinning_status,
    proxy_intercept_status,
    proxy_passthrough_reason,
    proxy_client_side_latency_ms,
    proxy_server_side_latency_ms,
    proxy_client_side_version,
    proxy_server_side_version,
    proxy_cert_verify,
    proxy_intercept_error,
    monitor_mirrored_pkts,
    monitor_mirrored_bytes,
    client_ip,
    client_ip_tags,
    client_port,
    client_os_desc,
    client_geolocation,
    client_country,
    client_super_administrative_area,
    client_administrative_area,
    client_sub_administrative_area,
    client_asn,
    subscriber_id,
    imei,
    imsi,
    phone_number,
    apn,
    server_ip,
    server_ip_tags,
    server_port,
    server_os_desc,
    server_geolocation,
    server_country,
    server_super_administrative_area,
    server_administrative_area,
    server_sub_administrative_area,
    server_asn,
    server_fqdn,
    server_fqdn_tags,
    server_domain,
    app_transition,
    app,
    app_category,
    app_debug_info,
    app_content,
    app_extra_info,
    fqdn_category_list,
    ip_protocol,
    decoded_path,
    dns_message_id,
    dns_qr,
    dns_opcode,
    dns_aa,
    dns_tc,
    dns_rd,
    dns_ra,
    dns_rcode,
    dns_qdcount,
    dns_ancount,
    dns_nscount,
    dns_arcount,
    dns_qname,
    dns_qtype,
    dns_qclass,
    dns_cname,
    dns_sub,
    dns_rr,
    dns_response_latency_ms,
    http_url,
    http_host,
    http_request_line,
    http_response_line,
    http_request_body,
    http_response_body,
    http_proxy_flag,
    http_sequence,
    http_cookie,
    http_referer,
    http_user_agent,
    http_request_content_length,
    http_request_content_type,
    http_response_content_length,
    http_response_content_type,
    http_set_cookie,
    http_version,
    http_status_code,
    http_response_latency_ms,
    http_session_duration_ms,
    http_action_file_size,
    ssl_version,
    ssl_sni,
    ssl_san,
    ssl_cn,
    ssl_handshake_latency_ms,
    ssl_ja3_hash,
    ssl_ja3s_hash,
    ssl_cert_issuer,
    ssl_cert_subject,
    ssl_esni_flag,
    ssl_ech_flag,
    dtls_cookie,
    dtls_version,
    dtls_sni,
    dtls_san,
    dtls_cn,
    dtls_handshake_latency_ms,
    dtls_ja3_fingerprint,
    dtls_ja3_hash,
    dtls_cert_issuer,
    dtls_cert_subject,
    mail_protocol_type,
    mail_account,
    mail_from_cmd,
    mail_to_cmd,
    mail_from,
    mail_password,
    mail_to,
    mail_cc,
    mail_bcc,
    mail_subject,
    mail_subject_charset,
    mail_attachment_name,
    mail_attachment_name_charset,
    mail_starttls_flag,
    mail_eml_file,
    ftp_account,
    ftp_url,
    ftp_link_type,
    quic_version,
    quic_sni,
    quic_user_agent,
    rdp_cookie,
    rdp_security_protocol,
    rdp_client_channels,
    rdp_keyboard_layout,
    rdp_client_version,
    rdp_client_name,
    rdp_client_product_id,
    rdp_desktop_width,
    rdp_desktop_height,
    rdp_requested_color_depth,
    rdp_certificate_type,
    rdp_certificate_count,
    rdp_certificate_permanent,
    rdp_encryption_level,
    rdp_encryption_method,
    ssh_version,
    ssh_auth_success,
    ssh_client_version,
    ssh_server_version,
    ssh_cipher_alg,
    ssh_mac_alg,
    ssh_compression_alg,
    ssh_kex_alg,
    ssh_host_key_alg,
    ssh_host_key,
    ssh_hassh,
    sip_call_id,
    sip_originator_description,
    sip_responder_description,
    sip_user_agent,
    sip_server,
    sip_originator_sdp_connect_ip,
    sip_originator_sdp_media_port,
    sip_originator_sdp_media_type,
    sip_originator_sdp_content,
    sip_responder_sdp_connect_ip,
    sip_responder_sdp_media_port,
    sip_responder_sdp_media_type,
    sip_responder_sdp_content,
    sip_duration_s,
    sip_bye,
    sip_bye_reason,
    rtp_payload_type_c2s,
    rtp_payload_type_s2c,
    rtp_pcap_path,
    rtp_originator_dir,
    stratum_cryptocurrency,
    stratum_mining_pools,
    stratum_mining_program,
    stratum_mining_subscribe,
    sent_pkts,
    received_pkts,
    sent_bytes,
    received_bytes,
    tcp_c2s_ip_fragments,
    tcp_s2c_ip_fragments,
    tcp_c2s_lost_bytes,
    tcp_s2c_lost_bytes,
    tcp_c2s_o3_pkts,
    tcp_s2c_o3_pkts,
    tcp_c2s_rtx_pkts,
    tcp_s2c_rtx_pkts,
    tcp_c2s_rtx_bytes,
    tcp_s2c_rtx_bytes,
    tcp_rtt_ms,
    tcp_client_isn,
    tcp_server_isn,
    packet_capture_file,
    in_src_mac,
    out_src_mac,
    in_dest_mac,
    out_dest_mac,
    encapsulation,
    dup_traffic_flag,
    tunnel_id_list,
    tunnel_endpoint_a_desc,
    tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.session_record_local
WHERE empty(security_rule_list) = 0
;

-- tsg_galaxy_v3.monitor_event_materialized_view
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster 
TO tsg_galaxy_v3.monitor_event_local
(
    recv_time Int64,
    log_id UInt64,
    decoded_as String,
    session_id UInt64,
    start_timestamp_ms DateTime64(3),
    end_timestamp_ms DateTime64(3),
    duration_ms Int32,
    tcp_handshake_latency_ms Nullable(Int32),
    ingestion_time Int64,
    processing_time Int64,
    -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
    device_id String,
    out_link_id Nullable(Int32),
    in_link_id Nullable(Int32),
    device_tag String,
    data_center String,
    device_group String,
    sled_ip String,
    address_type Int32,
    direction String,
    vsys_id Int32,
    t_vsys_id Int32,
    flags Int64,
    flags_identify_info String,
    c2s_ttl Nullable(Int32),
    s2c_ttl Nullable(Int32),
    security_rule_list Array(Int64),
    security_action String,
    monitor_rule_list Array(Int64),
    shaping_rule_list Array(Int64),
    proxy_rule_list Array(Int64),
    statistics_rule_list Array(Int64),
    sc_rule_list Array(Int64),
    sc_rsp_raw Array(Int64),
    sc_rsp_decrypted Array(Int64),
    proxy_action String,
    proxy_pinning_status Nullable(Int32),
    proxy_intercept_status Nullable(Int32),
    proxy_passthrough_reason String,
    proxy_client_side_latency_ms Nullable(Int32),
    proxy_server_side_latency_ms Nullable(Int32),
    proxy_client_side_version String,
    proxy_server_side_version String,
    proxy_cert_verify Nullable(Int32),
    proxy_intercept_error String,
    monitor_mirrored_pkts Nullable(Int32),
    monitor_mirrored_bytes Nullable(Int32),
    client_ip String,
    client_ip_tags Array(String),
    client_port Int32,
    client_os_desc String,
    client_geolocation LowCardinality(String),
    client_country String,
    client_super_administrative_area String,
    client_administrative_area String,
    client_sub_administrative_area String,
    client_asn Nullable(Int64),
    subscriber_id String,
    imei String,
    imsi String,
    phone_number String,
    apn String,
    server_ip String,
    server_ip_tags Array(String),
    server_port Int32,
    server_os_desc String,
    server_geolocation LowCardinality(String),
    server_country String,
    server_super_administrative_area String,
    server_administrative_area String,
    server_sub_administrative_area String,
    server_asn Nullable(Int64),
    server_fqdn String,
    server_fqdn_tags Array(String),
    server_domain String,
    app_transition String, 
    app LowCardinality(String),
    app_category String,
    app_debug_info String,
    app_content String,
    app_extra_info String,
    fqdn_category_list Array(Int64),
    ip_protocol LowCardinality(String),
    decoded_path LowCardinality(String),
    dns_message_id Nullable(Int32),
    dns_qr Nullable(Int32),
    dns_opcode Nullable(Int32),
    dns_aa Nullable(Int32),
    dns_tc Nullable(Int32),
    dns_rd Nullable(Int32),
    dns_ra Nullable(Int32),
    dns_rcode Nullable(Int32),
    dns_qdcount Nullable(Int32),
    dns_ancount Nullable(Int32),
    dns_nscount Nullable(Int32),
    dns_arcount Nullable(Int32),
    dns_qname String,
    dns_qtype Nullable(Int32),
    dns_qclass Nullable(Int32),
    dns_cname String,
    dns_sub Nullable(Int32),
    dns_rr String,
    dns_response_latency_ms Nullable(Int32),
    http_url String,
    http_host String,
    http_request_line String,
    http_response_line String,
    http_request_body String,
    http_response_body String,
    http_proxy_flag Nullable(Int32),
    http_sequence Nullable(Int32),
    http_cookie String,
    http_referer String,
    http_user_agent String,
    http_request_content_length Nullable(Int64),
    http_request_content_type String,
    http_response_content_length Nullable(Int64),
    http_response_content_type String,
    http_set_cookie String,
    http_version String,
    http_status_code Nullable(Int32),
    http_response_latency_ms Nullable(Int32),
    http_session_duration_ms Nullable(Int32),
    http_action_file_size Nullable(Int64),
    ssl_version String,
    ssl_sni String,
    ssl_san String,
    ssl_cn String,
    ssl_handshake_latency_ms Nullable(Int32),
    ssl_ja3_hash String,
    ssl_ja3s_hash String,
    ssl_cert_issuer String,
    ssl_cert_subject String,
    ssl_esni_flag Nullable(Int32),
    ssl_ech_flag Nullable(Int32),
    dtls_cookie String,
    dtls_version  String,
    dtls_sni String,
    dtls_san String,
    dtls_cn String,
    dtls_handshake_latency_ms Nullable(Int32),
    dtls_ja3_fingerprint String,
    dtls_ja3_hash String,
    dtls_cert_issuer String,
    dtls_cert_subject String,
    mail_protocol_type String,
    mail_account String,
    mail_from_cmd String,
    mail_to_cmd String,
    mail_from String,
    mail_password String,
    mail_to String,
    mail_cc String,
    mail_bcc String,
    mail_subject String,
    mail_subject_charset String,
    mail_attachment_name String,
    mail_attachment_name_charset String,
    mail_starttls_flag Nullable(Int32),
    mail_eml_file String,
    ftp_account String,
    ftp_url String,
    ftp_link_type String,
    quic_version String,
    quic_sni String,
    quic_user_agent String,
    rdp_cookie String,
    rdp_security_protocol String,
    rdp_client_channels String,
    rdp_keyboard_layout String,
    rdp_client_version String,
    rdp_client_name String,
    rdp_client_product_id String,
    rdp_desktop_width String,
    rdp_desktop_height String,
    rdp_requested_color_depth String,
    rdp_certificate_type String,
    rdp_certificate_count Nullable(Int32),
    rdp_certificate_permanent Nullable(Int32),
    rdp_encryption_level String,
    rdp_encryption_method String,
    ssh_version String,
    ssh_auth_success String,
    ssh_client_version String,
    ssh_server_version String,
    ssh_cipher_alg String,
    ssh_mac_alg String,
    ssh_compression_alg String,
    ssh_kex_alg String,
    ssh_host_key_alg String,
    ssh_host_key String,
    ssh_hassh String,
    sip_call_id String,
    sip_originator_description String,
    sip_responder_description String,
    sip_user_agent String,
    sip_server String,
    sip_originator_sdp_connect_ip String,
    sip_originator_sdp_media_port Nullable(Int32),
    sip_originator_sdp_media_type String,
    sip_originator_sdp_content String,
    sip_responder_sdp_connect_ip String,
    sip_responder_sdp_media_port Nullable(Int32),
    sip_responder_sdp_media_type String,
    sip_responder_sdp_content String,
    sip_duration_s Nullable(Int32),
    sip_bye String,
    sip_bye_reason String,
    rtp_payload_type_c2s Nullable(Int32),
    rtp_payload_type_s2c Nullable(Int32),
    rtp_pcap_path String,
    rtp_originator_dir Nullable(Int32),
    stratum_cryptocurrency String,
    stratum_mining_pools String,
    stratum_mining_program String,
    stratum_mining_subscribe String,
    sent_pkts Int64,
    received_pkts Int64,
    sent_bytes Int64,
    received_bytes Int64,
    tcp_c2s_ip_fragments Nullable(Int64),
    tcp_s2c_ip_fragments Nullable(Int64),
    tcp_c2s_lost_bytes Nullable(Int64),
    tcp_s2c_lost_bytes Nullable(Int64),
    tcp_c2s_o3_pkts Nullable(Int64),
    tcp_s2c_o3_pkts Nullable(Int64),
    tcp_c2s_rtx_pkts Nullable(Int64),
    tcp_s2c_rtx_pkts Nullable(Int64),
    tcp_c2s_rtx_bytes Nullable(Int64),
    tcp_s2c_rtx_bytes Nullable(Int64),
    tcp_rtt_ms Nullable(Int32),
    tcp_client_isn Nullable(Int64),
    tcp_server_isn Nullable(Int64),
    packet_capture_file String,
    in_src_mac String,
    out_src_mac String,
    in_dest_mac String,
    out_dest_mac String,
    encapsulation String,
    dup_traffic_flag Nullable(Int32),
    tunnel_id_list Array(Int64),
    tunnel_endpoint_a_desc String,
    tunnel_endpoint_b_desc String
) 
AS
SELECT
    recv_time,
    log_id,
    decoded_as,
    session_id,
    start_timestamp_ms,
    end_timestamp_ms,
    duration_ms,
    tcp_handshake_latency_ms,
    ingestion_time,
    processing_time,
    -- insert_time,
    device_id,
    out_link_id,
    in_link_id,
    device_tag,
    data_center,
    device_group,
    sled_ip,
    address_type,
    direction,
    vsys_id,
    t_vsys_id,
    flags,
    flags_identify_info,
    c2s_ttl,
    s2c_ttl,
    security_rule_list,
    security_action,
    monitor_rule_list,
    shaping_rule_list,
    proxy_rule_list,
    statistics_rule_list,
    sc_rule_list,
    sc_rsp_raw,
    sc_rsp_decrypted,
    proxy_action,
    proxy_pinning_status,
    proxy_intercept_status,
    proxy_passthrough_reason,
    proxy_client_side_latency_ms,
    proxy_server_side_latency_ms,
    proxy_client_side_version,
    proxy_server_side_version,
    proxy_cert_verify,
    proxy_intercept_error,
    monitor_mirrored_pkts,
    monitor_mirrored_bytes,
    client_ip,
    client_ip_tags,
    client_port,
    client_os_desc,
    client_geolocation,
    client_country,
    client_super_administrative_area,
    client_administrative_area,
    client_sub_administrative_area,
    client_asn,
    subscriber_id,
    imei,
    imsi,
    phone_number,
    apn,
    server_ip,
    server_ip_tags,
    server_port,
    server_os_desc,
    server_geolocation,
    server_country,
    server_super_administrative_area,
    server_administrative_area,
    server_sub_administrative_area,
    server_asn,
    server_fqdn,
    server_fqdn_tags,
    server_domain,
    app_transition,
    app,
    app_category,
    app_debug_info,
    app_content,
    app_extra_info,
    fqdn_category_list,
    ip_protocol,
    decoded_path,
    dns_message_id,
    dns_qr,
    dns_opcode,
    dns_aa,
    dns_tc,
    dns_rd,
    dns_ra,
    dns_rcode,
    dns_qdcount,
    dns_ancount,
    dns_nscount,
    dns_arcount,
    dns_qname,
    dns_qtype,
    dns_qclass,
    dns_cname,
    dns_sub,
    dns_rr,
    dns_response_latency_ms,
    http_url,
    http_host,
    http_request_line,
    http_response_line,
    http_request_body,
    http_response_body,
    http_proxy_flag,
    http_sequence,
    http_cookie,
    http_referer,
    http_user_agent,
    http_request_content_length,
    http_request_content_type,
    http_response_content_length,
    http_response_content_type,
    http_set_cookie,
    http_version,
    http_status_code,
    http_response_latency_ms,
    http_session_duration_ms,
    http_action_file_size,
    ssl_version,
    ssl_sni,
    ssl_san,
    ssl_cn,
    ssl_handshake_latency_ms,
    ssl_ja3_hash,
    ssl_ja3s_hash,
    ssl_cert_issuer,
    ssl_cert_subject,
    ssl_esni_flag,
    ssl_ech_flag,
    dtls_cookie,
    dtls_version,
    dtls_sni,
    dtls_san,
    dtls_cn,
    dtls_handshake_latency_ms,
    dtls_ja3_fingerprint,
    dtls_ja3_hash,
    dtls_cert_issuer,
    dtls_cert_subject,
    mail_protocol_type,
    mail_account,
    mail_from_cmd,
    mail_to_cmd,
    mail_from,
    mail_password,
    mail_to,
    mail_cc,
    mail_bcc,
    mail_subject,
    mail_subject_charset,
    mail_attachment_name,
    mail_attachment_name_charset,
    mail_starttls_flag,
    mail_eml_file,
    ftp_account,
    ftp_url,
    ftp_link_type,
    quic_version,
    quic_sni,
    quic_user_agent,
    rdp_cookie,
    rdp_security_protocol,
    rdp_client_channels,
    rdp_keyboard_layout,
    rdp_client_version,
    rdp_client_name,
    rdp_client_product_id,
    rdp_desktop_width,
    rdp_desktop_height,
    rdp_requested_color_depth,
    rdp_certificate_type,
    rdp_certificate_count,
    rdp_certificate_permanent,
    rdp_encryption_level,
    rdp_encryption_method,
    ssh_version,
    ssh_auth_success,
    ssh_client_version,
    ssh_server_version,
    ssh_cipher_alg,
    ssh_mac_alg,
    ssh_compression_alg,
    ssh_kex_alg,
    ssh_host_key_alg,
    ssh_host_key,
    ssh_hassh,
    sip_call_id,
    sip_originator_description,
    sip_responder_description,
    sip_user_agent,
    sip_server,
    sip_originator_sdp_connect_ip,
    sip_originator_sdp_media_port,
    sip_originator_sdp_media_type,
    sip_originator_sdp_content,
    sip_responder_sdp_connect_ip,
    sip_responder_sdp_media_port,
    sip_responder_sdp_media_type,
    sip_responder_sdp_content,
    sip_duration_s,
    sip_bye,
    sip_bye_reason,
    rtp_payload_type_c2s,
    rtp_payload_type_s2c,
    rtp_pcap_path,
    rtp_originator_dir,
    stratum_cryptocurrency,
    stratum_mining_pools,
    stratum_mining_program,
    stratum_mining_subscribe,
    sent_pkts,
    received_pkts,
    sent_bytes,
    received_bytes,
    tcp_c2s_ip_fragments,
    tcp_s2c_ip_fragments,
    tcp_c2s_lost_bytes,
    tcp_s2c_lost_bytes,
    tcp_c2s_o3_pkts,
    tcp_s2c_o3_pkts,
    tcp_c2s_rtx_pkts,
    tcp_s2c_rtx_pkts,
    tcp_c2s_rtx_bytes,
    tcp_s2c_rtx_bytes,
    tcp_rtt_ms,
    tcp_client_isn,
    tcp_server_isn,
    packet_capture_file,
    in_src_mac,
    out_src_mac,
    in_dest_mac,
    out_dest_mac,
    encapsulation,
    dup_traffic_flag,
    tunnel_id_list,
    tunnel_endpoint_a_desc,
    tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.session_record_local
WHERE empty(monitor_rule_list) = 0
;


CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record_local on cluster ck_cluster (
    log_id UInt64,
    recv_time Int64,
    vsys_id Int32,
    timestamp_us UInt64,
    egress_action Int32,
    job_id String,
    sled_ip String,
    device_group String,
    traffic_link_id Int32,
    source_ip String,
    source_port Nullable(Int32),
    destination_ip String,
    destination_port Nullable(Int32),
    packet String,
    packet_length Int32,
    measurements String
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id,job_id,recv_time,timestamp_us);

CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record on cluster ck_cluster (
    log_id UInt64,
    recv_time Int64,
    vsys_id Int32,
    timestamp_us UInt64,
    egress_action Int32,
    job_id String,
    sled_ip String,
    device_group String,
    traffic_link_id Int32,
    source_ip String,
    source_port Nullable(Int32),
    destination_ip String,
    destination_port Nullable(Int32),
    packet String,
    packet_length Int32,
    measurements String
)
ENGINE = Distributed('ck_cluster',
 'tsg_galaxy_v3',
 'datapath_telemetry_record_local',
 rand());

CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.traffic_sketch_metric_local on cluster ck_cluster
(
    log_id UInt64,
    recv_time Int64,
    vsys_id Int64,
    device_id String,
    device_group String,
    data_center String,
    direction String,
    ip_protocol String,
    client_ip String,
    server_ip String,
    internal_ip String,
    external_ip String,
    client_country String,
    server_country String,
    client_asn Nullable(Int64),
    server_asn Nullable(Int64),
    server_fqdn String,
    server_domain String,
    app String,
    app_category String,
    c2s_ttl Nullable(Int32),
    s2c_ttl Nullable(Int32),
    c2s_link_id Nullable(Int32),
    s2c_link_id Nullable(Int32),
    sessions Int64,
    bytes Int64,
    sent_bytes Int64,
    received_bytes Int64,
    pkts Int64,
    sent_pkts Int64,
    received_pkts Int64,
    asymmetric_c2s_flows Int64,
    asymmetric_s2c_flows Int64,
    c2s_fragments Int64,
    s2c_fragments Int64,
    c2s_tcp_lost_bytes Int64,
    s2c_tcp_lost_bytes Int64,
    c2s_tcp_retransmitted_pkts Int64,
    s2c_tcp_retransmitted_pkts Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(recv_time))
ORDER BY (vsys_id,
 direction,
 ip_protocol,
 app,
 client_ip,
 recv_time);
 
 CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.traffic_sketch_metric on cluster ck_cluster
(
    log_id UInt64,
    recv_time Int64,
    vsys_id Int64,
    device_id String,
    device_group String,
    data_center String,
    direction String,
    ip_protocol String,
    client_ip String,
    server_ip String,
    internal_ip String,
    external_ip String,
    client_country String,
    server_country String,
    client_asn Nullable(Int64),
    server_asn Nullable(Int64),
    server_fqdn String,
    server_domain String,
    app String,
    app_category String,
    c2s_ttl Nullable(Int32),
    s2c_ttl Nullable(Int32),
    c2s_link_id Nullable(Int32),
    s2c_link_id Nullable(Int32),
    sessions Int64,
    bytes Int64,
    sent_bytes Int64,
    received_bytes Int64,
    pkts Int64,
    sent_pkts Int64,
    received_pkts Int64,
    asymmetric_c2s_flows Int64,
    asymmetric_s2c_flows Int64,
    c2s_fragments Int64,
    s2c_fragments Int64,
    c2s_tcp_lost_bytes Int64,
    s2c_tcp_lost_bytes Int64,
    c2s_tcp_retransmitted_pkts Int64,
    s2c_tcp_retransmitted_pkts Int64
)
ENGINE = Distributed('ck_cluster',
 'tsg_galaxy_v3',
 'traffic_sketch_metric_local',
 rand());