diff --git a/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql b/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql index ac3660c..4d25834 100644 --- a/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql +++ b/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql @@ -855,8 +855,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE -CREATE TABLE cyber_narrator_galaxy.metric_tag_local -( +CREATE TABLE cyber_narrator_galaxy.metric_tag_local ON CLUSTER ck_cluster ( tag String, stat_time Int64, ip_sketch String, diff --git a/cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl.sql b/cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl.sql index e33c6f6..aeefb2e 100644 --- a/cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl.sql +++ b/cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl.sql @@ -1269,8 +1269,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand()); -CREATE TABLE cyber_narrator_galaxy.metric_tag_local -( +CREATE TABLE cyber_narrator_galaxy.metric_tag_local ON CLUSTER ck_cluster ( tag String, stat_time Int64, ip_sketch String, diff --git a/cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_24.04.sql b/cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_24.04.sql new file mode 100644 index 0000000..aeefb2e --- /dev/null +++ b/cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_24.04.sql @@ -0,0 +1,2128 @@ +create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_cluster; +create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_query; + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUSTER ck_cluster ( + recv_time Int64, + log_id UInt64, + flags UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int64, + decoded_as String, + client_ip String, + server_ip String, + client_port Int64, + server_port Int64, + app String, + app_transition String, + decoded_path String, + ip_protocol LowCardinality(String), + l7_protocol String, + out_link_id Nullable(Int64), + in_link_id Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + http_host String, + http_url String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_line String, + http_response_line String, + http_status_code Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Nullable(Int64), + dns_rcode Nullable(Int64), + dns_qname String, + dns_qtype Nullable(Int64), + dns_qclass Nullable(Int64), + dns_sub Nullable(Int64), + dns_rr String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + out_link_direction String, + in_link_direction String, + domain String, + domain_sld String, + domain_category_name String, + domain_category_group String, + domain_reputation_level String, + domain_icp_company_name String, + domain_whois_org String, + domain_tags Array(String), + client_zone String, + client_country_region String, + client_super_admin_area String, + client_admin_area String, + client_longitude Nullable(Float64), + client_latitude Nullable(Float64), + client_isp String, + client_asn String, + client_ip_tags Array(String), + server_zone String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_longitude Nullable(Float64), + server_latitude Nullable(Float64), + server_isp String, + server_asn String, + server_ip_tags Array(String), + app_category String, + app_subcategory String, + app_company String, + app_company_category String, + app_tags Array(String), + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64 DEFAULT 1, + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_rtt_ms Nullable(Int64), + http_response_latency_ms Nullable(Int64), + ssl_handshake_latency_ms Nullable(Int64), + dns_response_latency_ms Nullable(Int64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time)) ORDER BY recv_time SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_cluster ( + recv_time Int64, + log_id UInt64, + flags UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int64, + decoded_as String, + client_ip String, + server_ip String, + client_port Int64, + server_port Int64, + app String, + app_transition String, + decoded_path String, + ip_protocol LowCardinality(String), + l7_protocol String, + out_link_id Nullable(Int64), + in_link_id Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + http_host String, + http_url String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_line String, + http_response_line String, + http_status_code Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Nullable(Int64), + dns_rcode Nullable(Int64), + dns_qname String, + dns_qtype Nullable(Int64), + dns_qclass Nullable(Int64), + dns_sub Nullable(Int64), + dns_rr String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + out_link_direction String, + in_link_direction String, + domain String, + domain_sld String, + domain_category_name String, + domain_category_group String, + domain_reputation_level String, + domain_icp_company_name String, + domain_whois_org String, + domain_tags Array(String), + client_zone String, + client_country_region String, + client_super_admin_area String, + client_admin_area String, + client_longitude Nullable(Float64), + client_latitude Nullable(Float64), + client_isp String, + client_asn String, + client_ip_tags Array(String), + server_zone String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_longitude Nullable(Float64), + server_latitude Nullable(Float64), + server_isp String, + server_asn String, + server_ip_tags Array(String), + app_category String, + app_subcategory String, + app_company String, + app_company_category String, + app_tags Array(String), + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64 DEFAULT 1, + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_rtt_ms Nullable(Int64), + http_response_latency_ms Nullable(Int64), + ssl_handshake_latency_ms Nullable(Int64), + dns_response_latency_ms Nullable(Int64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_query ( + recv_time Int64, + log_id UInt64, + flags UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int64, + decoded_as String, + client_ip String, + server_ip String, + client_port Int64, + server_port Int64, + app String, + app_transition String, + decoded_path String, + ip_protocol LowCardinality(String), + l7_protocol String, + out_link_id Nullable(Int64), + in_link_id Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + http_host String, + http_url String, + http_cookie String, + http_referer String, + http_user_agent String, + http_request_line String, + http_response_line String, + http_status_code Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Nullable(Int64), + dns_rcode Nullable(Int64), + dns_qname String, + dns_qtype Nullable(Int64), + dns_qclass Nullable(Int64), + dns_sub Nullable(Int64), + dns_rr String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + out_link_direction String, + in_link_direction String, + domain String, + domain_sld String, + domain_category_name String, + domain_category_group String, + domain_reputation_level String, + domain_icp_company_name String, + domain_whois_org String, + domain_tags Array(String), + client_zone String, + client_country_region String, + client_super_admin_area String, + client_admin_area String, + client_longitude Nullable(Float64), + client_latitude Nullable(Float64), + client_isp String, + client_asn String, + client_ip_tags Array(String), + server_zone String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_longitude Nullable(Float64), + server_latitude Nullable(Float64), + server_isp String, + server_asn String, + server_ip_tags Array(String), + app_category String, + app_subcategory String, + app_company String, + app_company_category String, + app_tags Array(String), + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64 DEFAULT 1, + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_rtt_ms Nullable(Int64), + http_response_latency_ms Nullable(Int64), + ssl_handshake_latency_ms Nullable(Int64), + dns_response_latency_ms Nullable(Int64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_local ON CLUSTER ck_cluster ( + ip String, + side String, + zone String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64), + asymmetric_sessions Int64, + bulky_sessions Int64, + cbr_streaming_sessions Int64, + download_sessions Int64, + interactive_sessions Int64, + pseudo_unidirectional_sessions Int64, + streaming_sessions Int64, + unidirectional_sessions Int64, + random_looking_sessions Int64, + bidirectional_sessions Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,ip) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip ON CLUSTER ck_cluster ( + ip String, + side String, + zone String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64), + asymmetric_sessions Int64, + bulky_sessions Int64, + cbr_streaming_sessions Int64, + download_sessions Int64, + interactive_sessions Int64, + pseudo_unidirectional_sessions Int64, + streaming_sessions Int64, + unidirectional_sessions Int64, + random_looking_sessions Int64, + bidirectional_sessions Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_ip_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip ON CLUSTER ck_query ( + ip String, + side String, + zone String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64), + asymmetric_sessions Int64, + bulky_sessions Int64, + cbr_streaming_sessions Int64, + download_sessions Int64, + interactive_sessions Int64, + pseudo_unidirectional_sessions Int64, + streaming_sessions Int64, + unidirectional_sessions Int64, + random_looking_sessions Int64, + bidirectional_sessions Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_ip_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_region_local ON CLUSTER ck_cluster ( + country_region String, + super_admin_area String, + admin_area String, + side String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,country_region,super_admin_area,admin_area) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_region ON CLUSTER ck_cluster ( + country_region String, + super_admin_area String, + admin_area String, + side String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_region_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_region ON CLUSTER ck_query ( + country_region String, + super_admin_area String, + admin_area String, + side String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_region_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_asn_local ON CLUSTER ck_cluster ( + asn String, + isp String, + side String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,asn,isp) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_asn ON CLUSTER ck_cluster ( + asn String, + isp String, + side String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_asn_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_asn ON CLUSTER ck_query ( + asn String, + isp String, + side String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_asn_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_application_local ON CLUSTER ck_cluster ( + app String, + app_category String, + app_subcategory String, + app_company String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,app,app_category,app_subcategory,app_company) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_application ON CLUSTER ck_cluster ( + app String, + app_category String, + app_subcategory String, + app_company String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_application_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_application ON CLUSTER ck_query ( + app String, + app_category String, + app_subcategory String, + app_company String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_application_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_domain_local ON CLUSTER ck_cluster ( + domain String, + domain_sld String, + domain_category_name String, + domain_category_group String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,domain,domain_category_name,domain_category_group) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_domain ON CLUSTER ck_cluster ( + domain String, + domain_sld String, + domain_category_name String, + domain_category_group String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_domain_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_domain ON CLUSTER ck_query ( + domain String, + domain_sld String, + domain_category_name String, + domain_category_group String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_domain_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_protocol_local ON CLUSTER ck_cluster ( + l7_protocol String, + server_port Int64, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,l7_protocol,server_port) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_protocol ON CLUSTER ck_cluster ( + l7_protocol String, + server_port Int64, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_protocol_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_protocol ON CLUSTER ck_query ( + l7_protocol String, + server_port Int64, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_protocol_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_link_local ON CLUSTER ck_cluster ( + client_country_region String, + client_super_admin_area String, + client_admin_area String, + client_zone String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_zone String, + out_link_id Int64, + in_link_id Int64, + out_link_direction String, + in_link_direction String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_link ON CLUSTER ck_cluster ( + client_country_region String, + client_super_admin_area String, + client_admin_area String, + client_zone String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_zone String, + out_link_id Int64, + in_link_id Int64, + out_link_direction String, + in_link_direction String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_link_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_link ON CLUSTER ck_query ( + client_country_region String, + client_super_admin_area String, + client_admin_area String, + client_zone String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_zone String, + out_link_id Int64, + in_link_id Int64, + out_link_direction String, + in_link_direction String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_link_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app_local ON CLUSTER ck_cluster ( + subscriber_id String, + app String, + imei String, + imsi String, + phone_number String, + apn String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id,app) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTER ck_cluster ( + subscriber_id String, + app String, + imei String, + imsi String, + phone_number String, + apn String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTER ck_query ( + subscriber_id String, + app String, + imei String, + imsi String, + phone_number String, + apn String, + stat_time Int64, + sent_pkts Int64, + sent_bytes Int64, + received_pkts Int64, + received_bytes Int64, + sessions Int64, + traffic_inbound_byte Int64, + traffic_inbound_pkt Int64, + traffic_outbound_byte Int64, + traffic_outbound_pkt Int64, + traffic_internal_byte Int64, + traffic_internal_pkt Int64, + traffic_through_byte Int64, + traffic_through_pkt Int64, + tcp_c2s_lost_bytes_ratio Nullable(Float64), + tcp_s2c_lost_bytes_ratio Nullable(Float64), + tcp_lost_bytes_ratio Nullable(Float64), + tcp_c2s_o3_pkts_ratio Nullable(Float64), + tcp_s2c_o3_pkts_ratio Nullable(Float64), + tcp_o3_pkts_ratio Nullable(Float64), + tcp_c2s_rtx_bytes_ratio Nullable(Float64), + tcp_s2c_rtx_bytes_ratio Nullable(Float64), + tcp_rtx_bytes_ratio Nullable(Float64), + tcp_c2s_rtx_pkts_ratio Nullable(Float64), + tcp_s2c_rtx_pkts_ratio Nullable(Float64), + tcp_rtx_pkts_ratio Nullable(Float64), + avg_tcp_rtt_ms Nullable(Float64), + avg_http_response_latency_ms Nullable(Float64), + avg_ssl_handshake_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand()); + + +CREATE TABLE cyber_narrator_galaxy.metric_tag_local ON CLUSTER ck_cluster ( + tag String, + stat_time Int64, + ip_sketch String, + domain_sketch String, + ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch), + domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch) +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(stat_time)) +ORDER BY (stat_time, tag); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_cluster ( + tag String, + stat_time Int64, + ip_sketch String, + domain_sketch String, + ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch), + domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_query ( + tag String, + stat_time Int64, + ip_sketch String, + domain_sketch String, + ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch), + domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster ( + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), + first_location String, + second_location String, + third_location String, + data_source String, + stat_time Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster ( + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), + first_location String, + second_location String, + third_location String, + data_source String, + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_query ( + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), + first_location String, + second_location String, + third_location String, + data_source String, + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_server_ip_local ON CLUSTER ck_cluster ( + server_ip String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_isp String, + server_role Array(String), + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,server_ip) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_server_ip ON CLUSTER ck_cluster ( + server_ip String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_isp String, + server_role Array(String), + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_server_ip_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_server_ip ON CLUSTER ck_query ( + server_ip String, + server_country_region String, + server_super_admin_area String, + server_admin_area String, + server_isp String, + server_role Array(String), + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_server_ip_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qname_local ON CLUSTER ck_cluster ( + qname String, + qname_sld String, + qname_tld String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,qname) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qname ON CLUSTER ck_cluster ( + qname String, + qname_sld String, + qname_tld String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_qname_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qname ON CLUSTER ck_query ( + qname String, + qname_sld String, + qname_tld String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_qname_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qtype_local ON CLUSTER ck_cluster ( + qtype Int64, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,qtype) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qtype ON CLUSTER ck_cluster ( + qtype Int64, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_qtype_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qtype ON CLUSTER ck_query ( + qtype Int64, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_qtype_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rcode_local ON CLUSTER ck_cluster ( + rcode Int64, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,rcode) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rcode ON CLUSTER ck_cluster ( + rcode Int64, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rcode_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rcode ON CLUSTER ck_query ( + rcode Int64, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rcode_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_a_local ON CLUSTER ck_cluster ( + rr_a String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,rr_a) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_a ON CLUSTER ck_cluster ( + rr_a String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_a_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_a ON CLUSTER ck_query ( + rr_a String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_a_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_aaaa_local ON CLUSTER ck_cluster ( + rr_aaaa String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,rr_aaaa) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_aaaa ON CLUSTER ck_cluster ( + rr_aaaa String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_aaaa_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_aaaa ON CLUSTER ck_query ( + rr_aaaa String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_aaaa_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_cname_local ON CLUSTER ck_cluster ( + rr_cname String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,rr_cname) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_cname ON CLUSTER ck_cluster ( + rr_cname String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_cname_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_cname ON CLUSTER ck_query ( + rr_cname String, + stat_time Int64, + query_num Int64, + traffic_inbound_byte Int64, + traffic_outbound_byte Int64, + internal_query_num Int64, + external_query_num Int64, + avg_response_latency_ms Nullable(Float64) +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_cname_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local ON CLUSTER ck_cluster ( + app_name String, + domain String, + ip String, + ip_country_region String, + ip_super_admin_area String, + ip_admin_area String, + ip_asn String, + ip_isp String, + domain_category_name String, + domain_category_group String, + app_category String, + app_subcategory String, + entity_tags Array(String), + stat_time Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_cluster ( + app_name String, + domain String, + ip String, + ip_country_region String, + ip_super_admin_area String, + ip_admin_area String, + ip_asn String, + ip_isp String, + domain_category_name String, + domain_category_group String, + app_category String, + app_subcategory String, + entity_tags Array(String), + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_query ( + app_name String, + domain String, + ip String, + ip_country_region String, + ip_super_admin_area String, + ip_admin_area String, + ip_asn String, + ip_isp String, + domain_category_name String, + domain_category_group String, + app_category String, + app_subcategory String, + entity_tags Array(String), + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local ON CLUSTER ck_cluster ( + ip String, + l7_protocol String, + port Int64, + stat_time Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_cluster ( + ip String, + l7_protocol String, + port Int64, + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_query ( + ip String, + l7_protocol String, + port Int64, + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app_local on cluster ck_cluster +( + app_name String, + domain String, + ip String, + ip_country_region String, + ip_super_admin_area String, + ip_admin_area String, + ip_asn String, + ip_isp String, + domain_category_name String, + domain_category_group String, + app_category String, + app_subcategory String, + entity_tags Array(String), + create_time Int64, + update_time Int64 +) +ENGINE = MergeTree +ORDER BY (ip, + domain, + app_name) +TTL toDateTime(update_time) + toIntervalSecond(2592000), + toDateTime(update_time) + toIntervalSecond(1) GROUP BY ip, + domain, + app_name SET create_time = min(create_time), + update_time = max(update_time), + ip_country_region = anyLast(ip_country_region), + ip_super_admin_area = anyLast(ip_super_admin_area), + ip_admin_area = anyLast(ip_admin_area), + ip_asn = anyLast(ip_asn), + ip_isp = anyLast(ip_isp), + domain_category_name = anyLast(domain_category_name), + domain_category_group = anyLast(domain_category_group), + app_category = anyLast(app_category), + app_subcategory = anyLast(app_subcategory), + entity_tags = groupUniqArrayArray(entity_tags); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_query +( + app_name String, + domain String, + ip String, + ip_country_region String, + ip_super_admin_area String, + ip_admin_area String, + ip_asn String, + ip_isp String, + domain_category_name String, + domain_category_group String, + app_category String, + app_subcategory String, + entity_tags Array(String), + create_time Int64, + update_time Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'session_relation_domain_ip_app_local', + rand()); + + CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_cluster +( + app_name String, + domain String, + ip String, + ip_country_region String, + ip_super_admin_area String, + ip_admin_area String, + ip_asn String, + ip_isp String, + domain_category_name String, + domain_category_group String, + app_category String, + app_subcategory String, + entity_tags Array(String), + create_time Int64, + update_time Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'session_relation_domain_ip_app_local', + rand()); + + + CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app_local ON CLUSTER ck_cluster ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + create_time Int64, + update_time Int64 +) ENGINE = MergeTree + ORDER BY (subscriber_id, app_name) + TTL toDateTime(update_time) + toIntervalSecond(2592000), + toDateTime(update_time) + toIntervalSecond(1) + GROUP BY subscriber_id, app_name + SET create_time = min(create_time), + update_time = max(update_time), + imei = anyLast(imei), + imsi = anyLast(imsi), + phone_number = anyLast(phone_number), + apn = anyLast(apn), + app_category = anyLast(app_category), + app_subcategory = anyLast(app_subcategory); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_cluster ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + create_time Int64, + update_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_query ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + create_time Int64, + update_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app_local ON CLUSTER ck_cluster ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + stat_time Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_cluster ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_query ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand()); + + + CREATE TABLE if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_local on cluster ck_cluster +( + ip String, + l7_protocol String, + port Int64, + create_time Int64, + update_time Int64 +) +ENGINE = MergeTree +ORDER BY (ip, + port, + l7_protocol) +TTL toDateTime(update_time) + toIntervalSecond(2592000), + toDateTime(update_time) + toIntervalSecond(1) GROUP BY ip, + port, + l7_protocol SET create_time = min(create_time), + update_time = max(update_time); + + CREATE TABLE if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute on cluster ck_query +( + ip String, + l7_protocol String, + port Int64, + create_time Int64, + update_time Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'cn_ip_dynamic_attribute_local', + rand()); + + CREATE TABLE if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute on cluster ck_cluster +( + ip String, + l7_protocol String, + port Int64, + create_time Int64, + update_time Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'cn_ip_dynamic_attribute_local', + rand()); + + + CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event_local ON CLUSTER ck_cluster +( + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + victim_country_region String, + victim_super_admin_area String, + victim_admin_area String, + victim_longitude Nullable(Float64), + victim_latitude Nullable(Float64), + offender_ip String, + offender_country_region String, + offender_super_admin_area String, + offender_admin_area String, + offender_longitude Nullable(Float64), + offender_latitude Nullable(Float64), + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = MergeTree +ORDER BY event_id +TTL toDateTime(end_time) + toIntervalSecond(2592000), + toDateTime(end_time) + toIntervalSecond(1) GROUP BY event_id + SET + status = anyLast(status), + is_builtin = anyLast(is_builtin), + rule_type = anyLast(rule_type), + victim_ip = anyLast(victim_ip), + victim_country_region = anyLast(victim_country_region), + victim_super_admin_area = anyLast(victim_super_admin_area), + victim_admin_area = anyLast(victim_admin_area), + victim_longitude = anyLast(victim_longitude), + victim_latitude = anyLast(victim_latitude), + offender_ip = anyLast(offender_ip), + offender_country_region = anyLast(offender_country_region), + offender_super_admin_area = anyLast(offender_super_admin_area), + offender_admin_area = anyLast(offender_admin_area), + offender_longitude = anyLast(offender_longitude), + offender_latitude = anyLast(offender_latitude), + event_info = anyLast(event_info), + event_key = anyLast(event_key), + severity = anyLast(severity), + event_type = anyLast(event_type), + duration_s = anyLast(duration_s), + event_name = anyLast(event_name), + app = anyLast(app), + domain = anyLast(domain), + rule_id = anyLast(rule_id), + start_time = anyLast(start_time), + end_time = max(end_time), + match_times = anyLast(match_times); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_cluster +( + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + victim_country_region String, + victim_super_admin_area String, + victim_admin_area String, + victim_longitude Nullable(Float64), + victim_latitude Nullable(Float64), + offender_ip String, + offender_country_region String, + offender_super_admin_area String, + offender_admin_area String, + offender_longitude Nullable(Float64), + offender_latitude Nullable(Float64), + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'cn_security_event_local', + rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_query +( + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + victim_country_region String, + victim_super_admin_area String, + victim_admin_area String, + victim_longitude Nullable(Float64), + victim_latitude Nullable(Float64), + offender_ip String, + offender_country_region String, + offender_super_admin_area String, + offender_admin_area String, + offender_longitude Nullable(Float64), + offender_latitude Nullable(Float64), + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'cn_security_event_local', + rand()); + + + CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric on cluster ck_cluster +TO cyber_narrator_galaxy.session_relation_domain_ip_app_local +( + app_name String, + domain String, + ip String, + ip_country_region String, + ip_super_admin_area String, + ip_admin_area String, + ip_asn String, + ip_isp String, + domain_category_name String, + domain_category_group String, + app_category String, + app_subcategory String, + entity_tags Array(String), + create_time Int64, + update_time Int64 +) AS +SELECT + app_name AS app_name, + domain AS domain, + ip AS ip, + anyLast(ip_country_region) AS ip_country_region, + anyLast(ip_super_admin_area) AS ip_super_admin_area, + anyLast(ip_admin_area) AS ip_admin_area, + anyLast(ip_asn) AS ip_asn, + anyLast(ip_isp) AS ip_isp, + anyLast(domain_category_name) AS domain_category_name, + anyLast(domain_category_group) AS domain_category_group, + anyLast(app_category) AS app_category, + anyLast(app_subcategory) AS app_subcategory, + groupUniqArrayArray(entity_tags) AS entity_tags, + min(c1.stat_time) AS create_time, + max(c1.stat_time) AS update_time +FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local AS c1 +GROUP BY + ip, + app_name, + domain; + +CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_subscriber_app_view_metric on cluster ck_cluster +TO cyber_narrator_galaxy.session_relation_subscriber_app_local +( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + create_time Int64, + update_time Int64 +) AS +SELECT + app_name AS app_name, + subscriber_id AS subscriber_id, + anyLast(imei) AS imei, + anyLast(imsi) AS imsi, + anyLast(phone_number) AS phone_number, + anyLast(apn) AS apn, + anyLast(app_category) AS app_category, + anyLast(app_subcategory) AS app_subcategory, + min(c1.stat_time) AS create_time, + max(c1.stat_time) AS update_time +FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app_local AS c1 +GROUP BY + subscriber_id, + app_name; + +CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric on cluster ck_cluster +TO cyber_narrator_galaxy.cn_ip_dynamic_attribute_local +( + ip String, + l7_protocol String, + port Int64, + create_time Int64, + update_time Int64 +) AS +SELECT + ip AS ip, + l7_protocol AS l7_protocol, + port AS port, + min(c1.stat_time) AS create_time, + max(c1.stat_time) AS update_time +FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local AS c1 +GROUP BY + ip, + l7_protocol, + port; diff --git a/cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_check_24.04.sql b/cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_check_24.04.sql index 4697cab..cb72177 100644 --- a/cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_check_24.04.sql +++ b/cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_check_24.04.sql @@ -1,14 +1,14 @@ -SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time -FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND start_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time