22.03 cn ck SQL

This commit is contained in:
zhanghongqing
2022-04-08 17:59:39 +08:00
parent 051471847d
commit 3d9132e907

View File

@@ -0,0 +1,613 @@
create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_cluster;
create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_query;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUSTER ck_cluster (
`common_recv_time` Int64,
`common_direction` Int64,
`common_stream_dir` Int64,
`common_schema_type` String,
`common_client_ip` String,
`common_server_ip` String,
`common_client_port` Int64,
`common_server_port` Int64,
`common_app_label` String,
`common_l4_protocol` String,
`common_l7_protocol` String,
`common_isp` String,
`common_egress_link_id` Int64,
`common_ingress_link_id` Int64,
`http_host` String,
`http_url` String,
`http_cookie` String,
`http_referer` String,
`http_user_agent` String,
`http_request_line` String,
`http_response_line` String,
`http_status_code` Int64,
`ssl_sni` String,
`ssl_san` String,
`ssl_ja3_fingerprint` String,
`ssl_ja3_hash` String,
`ssl_cert_issuer` String,
`ssl_cert_subject` String,
`dns_qr` Int64,
`dns_opcode` Int64,
`dns_aa` Int64,
`dns_rcode` Int64,
`dns_qname` String,
`dns_qtype` Int64,
`dns_qclass` Int64,
`dns_sub` Int64,
`dns_rr` String,
`ssh_version` String,
`ssh_auth_success` String,
`ssh_client_version` String,
`ssh_server_version` String,
`ssh_cipher_alg` String,
`ssh_mac_alg` String,
`ssh_compression_alg` String,
`ssh_kex_alg` String,
`ssh_host_key_alg` String,
`ssh_host_key` String,
`ssh_hassh` String,
`egress_link_direction` String,
`ingress_link_direction` String,
`domain` String,
`domain_category_name` String,
`domain_category_group` String,
`domain_reputation_score` Int64,
`domain_reputation_level` String,
`domain_icp_company_name` String,
`domain_whois_org` String,
`client_country` String,
`client_province` String,
`client_region` String,
`client_longitude` Float64,
`client_latitude` Float64,
`client_isp` String,
`server_country` String,
`server_province` String,
`server_region` String,
`server_longitude` Float64,
`server_latitude` Float64,
`server_isp` String,
`app_category` String,
`app_subcategory` String,
`app_company` String,
`app_company_category` String,
`app_risk` Int64,
`dns_server_role` Array(String),
`dns_server_org` String,
`dns_server_os` String,
`dns_server_software` String,
`dns_protocol` String,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_lantency_ms` Int64,
`ssl_con_latency_ms` Int64,
`dns_response_latency_ms` Int64
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY common_recv_time SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_cluster (
`common_recv_time` Int64,
`common_direction` Int64,
`common_stream_dir` Int64,
`common_schema_type` String,
`common_client_ip` String,
`common_server_ip` String,
`common_client_port` Int64,
`common_server_port` Int64,
`common_app_label` String,
`common_l4_protocol` String,
`common_l7_protocol` String,
`common_isp` String,
`common_egress_link_id` Int64,
`common_ingress_link_id` Int64,
`http_host` String,
`http_url` String,
`http_cookie` String,
`http_referer` String,
`http_user_agent` String,
`http_request_line` String,
`http_response_line` String,
`http_status_code` Int64,
`ssl_sni` String,
`ssl_san` String,
`ssl_ja3_fingerprint` String,
`ssl_ja3_hash` String,
`ssl_cert_issuer` String,
`ssl_cert_subject` String,
`dns_qr` Int64,
`dns_opcode` Int64,
`dns_aa` Int64,
`dns_rcode` Int64,
`dns_qname` String,
`dns_qtype` Int64,
`dns_qclass` Int64,
`dns_sub` Int64,
`dns_rr` String,
`ssh_version` String,
`ssh_auth_success` String,
`ssh_client_version` String,
`ssh_server_version` String,
`ssh_cipher_alg` String,
`ssh_mac_alg` String,
`ssh_compression_alg` String,
`ssh_kex_alg` String,
`ssh_host_key_alg` String,
`ssh_host_key` String,
`ssh_hassh` String,
`egress_link_direction` String,
`ingress_link_direction` String,
`domain` String,
`domain_category_name` String,
`domain_category_group` String,
`domain_reputation_score` Int64,
`domain_reputation_level` String,
`domain_icp_company_name` String,
`domain_whois_org` String,
`client_country` String,
`client_province` String,
`client_region` String,
`client_longitude` Float64,
`client_latitude` Float64,
`client_isp` String,
`server_country` String,
`server_province` String,
`server_region` String,
`server_longitude` Float64,
`server_latitude` Float64,
`server_isp` String,
`app_category` String,
`app_subcategory` String,
`app_company` String,
`app_company_category` String,
`app_risk` Int64,
`dns_server_role` Array(String),
`dns_server_org` String,
`dns_server_os` String,
`dns_server_software` String,
`dns_protocol` String,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_lantency_ms` Int64,
`ssl_con_latency_ms` Int64,
`dns_response_latency_ms` Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_query (
`common_recv_time` Int64,
`common_direction` Int64,
`common_stream_dir` Int64,
`common_schema_type` String,
`common_client_ip` String,
`common_server_ip` String,
`common_client_port` Int64,
`common_server_port` Int64,
`common_app_label` String,
`common_l4_protocol` String,
`common_l7_protocol` String,
`common_isp` String,
`common_egress_link_id` Int64,
`common_ingress_link_id` Int64,
`http_host` String,
`http_url` String,
`http_cookie` String,
`http_referer` String,
`http_user_agent` String,
`http_request_line` String,
`http_response_line` String,
`http_status_code` Int64,
`ssl_sni` String,
`ssl_san` String,
`ssl_ja3_fingerprint` String,
`ssl_ja3_hash` String,
`ssl_cert_issuer` String,
`ssl_cert_subject` String,
`dns_qr` Int64,
`dns_opcode` Int64,
`dns_aa` Int64,
`dns_rcode` Int64,
`dns_qname` String,
`dns_qtype` Int64,
`dns_qclass` Int64,
`dns_sub` Int64,
`dns_rr` String,
`ssh_version` String,
`ssh_auth_success` String,
`ssh_client_version` String,
`ssh_server_version` String,
`ssh_cipher_alg` String,
`ssh_mac_alg` String,
`ssh_compression_alg` String,
`ssh_kex_alg` String,
`ssh_host_key_alg` String,
`ssh_host_key` String,
`ssh_hassh` String,
`egress_link_direction` String,
`ingress_link_direction` String,
`domain` String,
`domain_category_name` String,
`domain_category_group` String,
`domain_reputation_score` Int64,
`domain_reputation_level` String,
`domain_icp_company_name` String,
`domain_whois_org` String,
`client_country` String,
`client_province` String,
`client_region` String,
`client_longitude` Float64,
`client_latitude` Float64,
`client_isp` String,
`server_country` String,
`server_province` String,
`server_region` String,
`server_longitude` Float64,
`server_latitude` Float64,
`server_isp` String,
`app_category` String,
`app_subcategory` String,
`app_company` String,
`app_company_category` String,
`app_risk` Int64,
`dns_server_role` Array(String),
`dns_server_org` String,
`dns_server_os` String,
`dns_server_software` String,
`dns_protocol` String,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_lantency_ms` Int64,
`ssl_con_latency_ms` Int64,
`dns_response_latency_ms` Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.security_event_local ON CLUSTER ck_cluster (
`event_id` UInt64,
`start_time` Int64,
`event_severity` String,
`security_type` String,
`victim_ip` String,
`victim_location_country` String,
`victim_location_province` String,
`victim_location_region` String,
`victim_asn` String,
`offender_ip` String,
`offender_location_country` String,
`offender_location_province` String,
`offender_location_region` String,
`offender_asn` String,
`client_ip` String,
`server_ip` String,
`domain` String,
`domain_category_name` String,
`domain_category_group` String,
`domain_reputation_score` String,
`domain_reputation_level` String,
`app_name` String,
`app_category` String,
`app_subcategory` String,
`app_risk` Int64,
`malware_name` String,
`malware_alias` Array(String),
`malware_description` String,
`malware_platforms` Array(String),
`malware_techniques` Array(String),
`malware_groups` Array(String),
`phishing_url` String,
`phishing_brand_name` String,
`phishing_brand_industry` String,
`phishing_brand_website` String,
`phishing_language` String,
`cryptomining_pool` String,
`cryptomining_coin_type` String,
`cryptomining_software` String,
`dga_family` String,
`ddos_type` String,
`ddos_conditions` String,
`ddos_source_list` String
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(start_time)) ORDER BY (event_id, start_time) SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.security_event ON CLUSTER ck_cluster (
`event_id` UInt64,
`start_time` Int64,
`event_severity` String,
`security_type` String,
`victim_ip` String,
`victim_location_country` String,
`victim_location_province` String,
`victim_location_region` String,
`victim_asn` String,
`offender_ip` String,
`offender_location_country` String,
`offender_location_province` String,
`offender_location_region` String,
`offender_asn` String,
`client_ip` String,
`server_ip` String,
`domain` String,
`domain_category_name` String,
`domain_category_group` String,
`domain_reputation_score` String,
`domain_reputation_level` String,
`app_name` String,
`app_category` String,
`app_subcategory` String,
`app_risk` Int64,
`malware_name` String,
`malware_alias` Array(String),
`malware_description` String,
`malware_platforms` Array(String),
`malware_techniques` Array(String),
`malware_groups` Array(String),
`phishing_url` String,
`phishing_brand_name` String,
`phishing_brand_industry` String,
`phishing_brand_website` String,
`phishing_language` String,
`cryptomining_pool` String,
`cryptomining_coin_type` String,
`cryptomining_software` String,
`dga_family` String,
`ddos_type` String,
`ddos_conditions` String,
`ddos_source_list` String
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'security_event_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.security_event ON CLUSTER ck_query (
`event_id` UInt64,
`start_time` Int64,
`event_severity` String,
`security_type` String,
`victim_ip` String,
`victim_location_country` String,
`victim_location_province` String,
`victim_location_region` String,
`victim_asn` String,
`offender_ip` String,
`offender_location_country` String,
`offender_location_province` String,
`offender_location_region` String,
`offender_asn` String,
`client_ip` String,
`server_ip` String,
`domain` String,
`domain_category_name` String,
`domain_category_group` String,
`domain_reputation_score` String,
`domain_reputation_level` String,
`app_name` String,
`app_category` String,
`app_subcategory` String,
`app_risk` Int64,
`malware_name` String,
`malware_alias` Array(String),
`malware_description` String,
`malware_platforms` Array(String),
`malware_techniques` Array(String),
`malware_groups` Array(String),
`phishing_url` String,
`phishing_brand_name` String,
`phishing_brand_industry` String,
`phishing_brand_website` String,
`phishing_language` String,
`cryptomining_pool` String,
`cryptomining_coin_type` String,
`cryptomining_software` String,
`dga_family` String,
`ddos_type` String,
`ddos_conditions` String,
`ddos_source_list` String
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'security_event_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_map_local ON CLUSTER ck_cluster (
`server_country` String,
`client_country` String,
`server_province` String,
`client_province` String,
`server_region` String,
`client_region` String,
`common_recv_time` Int64,
`common_schema_type` String,
`common_l4_protocol` String,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_latency_ms` Int64,
`ssl_con_latency_ms` Int64
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_recv_time, server_country, common_l4_protocol, common_schema_type, server_region) SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_map ON CLUSTER ck_cluster (
`server_country` String,
`client_country` String,
`server_province` String,
`client_province` String,
`server_region` String,
`client_region` String,
`common_recv_time` Int64,
`common_schema_type` String,
`common_l4_protocol` String,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_latency_ms` Int64,
`ssl_con_latency_ms` Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_map_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_map ON CLUSTER ck_query (
`server_country` String,
`client_country` String,
`server_province` String,
`client_province` String,
`server_region` String,
`client_region` String,
`common_recv_time` Int64,
`common_schema_type` String,
`common_l4_protocol` String,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_latency_ms` Int64,
`ssl_con_latency_ms` Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_map_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_top_entity_local ON CLUSTER ck_cluster (
`common_client_ip` String,
`common_server_ip` String,
`domain` String,
`common_app_label` LowCardinality(String),
`common_recv_time` Int64,
`group_by` LowCardinality(String),
`order_by` LowCardinality(String),
`time_granularity` Int64,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_latency_ms` Int64,
`ssl_con_latency_ms` Int64,
`packet_loss` Float64,
`packet_retrans` Float64
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (group_by, order_by, common_recv_time, time_granularity) SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_top_entity ON CLUSTER ck_cluster (
`common_client_ip` String,
`common_server_ip` String,
`domain` String,
`common_app_label` LowCardinality(String),
`common_recv_time` Int64,
`group_by` LowCardinality(String),
`order_by` LowCardinality(String),
`time_granularity` Int64,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_latency_ms` Int64,
`ssl_con_latency_ms` Int64,
`packet_loss` Float64,
`packet_retrans` Float64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_top_entity_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_top_entity ON CLUSTER ck_query (
`common_client_ip` String,
`common_server_ip` String,
`domain` String,
`common_app_label` LowCardinality(String),
`common_recv_time` Int64,
`group_by` LowCardinality(String),
`order_by` LowCardinality(String),
`time_granularity` Int64,
`common_c2s_pkt_num` Int64,
`common_c2s_byte_num` Int64,
`common_s2c_pkt_num` Int64,
`common_s2c_byte_num` Int64,
`common_sessions` Int64,
`common_c2s_tcp_lostlen` Int64,
`common_s2c_tcp_lostlen` Int64,
`common_c2s_tcp_unorder_num` Int64,
`common_s2c_tcp_unorder_num` Int64,
`common_c2s_byte_retrans` Int64,
`common_s2c_byte_retrans` Int64,
`common_c2s_pkt_retrans` Int64,
`common_s2c_pkt_retrans` Int64,
`common_establish_latency_ms` Int64,
`http_response_latency_ms` Int64,
`ssl_con_latency_ms` Int64,
`packet_loss` Float64,
`packet_retrans` Float64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_top_entity_local', rand());