From 3d9132e907fda65f6a1257498944c59d4e2ed507 Mon Sep 17 00:00:00 2001 From: zhanghongqing Date: Fri, 8 Apr 2022 17:59:39 +0800 Subject: [PATCH] 22.03 cn ck SQL --- .../Clickhouse_CN_建表语句.sql | 613 ++++++++++++++++++ 1 file changed, 613 insertions(+) create mode 100644 Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql new file mode 100644 index 0000000..7384f25 --- /dev/null +++ b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql @@ -0,0 +1,613 @@ +create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_cluster; +create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_query; + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUSTER ck_cluster ( +`common_recv_time` Int64, + `common_direction` Int64, + `common_stream_dir` Int64, + `common_schema_type` String, + `common_client_ip` String, + `common_server_ip` String, + `common_client_port` Int64, + `common_server_port` Int64, + `common_app_label` String, + `common_l4_protocol` String, + `common_l7_protocol` String, + `common_isp` String, + `common_egress_link_id` Int64, + `common_ingress_link_id` Int64, + `http_host` String, + `http_url` String, + `http_cookie` String, + `http_referer` String, + `http_user_agent` String, + `http_request_line` String, + `http_response_line` String, + `http_status_code` Int64, + `ssl_sni` String, + `ssl_san` String, + `ssl_ja3_fingerprint` String, + `ssl_ja3_hash` String, + `ssl_cert_issuer` String, + `ssl_cert_subject` String, + `dns_qr` Int64, + `dns_opcode` Int64, + `dns_aa` Int64, + `dns_rcode` Int64, + `dns_qname` String, + `dns_qtype` Int64, + `dns_qclass` Int64, + `dns_sub` Int64, + `dns_rr` String, + `ssh_version` String, + `ssh_auth_success` String, + `ssh_client_version` String, + `ssh_server_version` String, + `ssh_cipher_alg` String, + `ssh_mac_alg` String, + `ssh_compression_alg` String, + `ssh_kex_alg` String, + `ssh_host_key_alg` String, + `ssh_host_key` String, + `ssh_hassh` String, + `egress_link_direction` String, + `ingress_link_direction` String, + `domain` String, + `domain_category_name` String, + `domain_category_group` String, + `domain_reputation_score` Int64, + `domain_reputation_level` String, + `domain_icp_company_name` String, + `domain_whois_org` String, + `client_country` String, + `client_province` String, + `client_region` String, + `client_longitude` Float64, + `client_latitude` Float64, + `client_isp` String, + `server_country` String, + `server_province` String, + `server_region` String, + `server_longitude` Float64, + `server_latitude` Float64, + `server_isp` String, + `app_category` String, + `app_subcategory` String, + `app_company` String, + `app_company_category` String, + `app_risk` Int64, + `dns_server_role` Array(String), + `dns_server_org` String, + `dns_server_os` String, + `dns_server_software` String, + `dns_protocol` String, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_lantency_ms` Int64, + `ssl_con_latency_ms` Int64, + `dns_response_latency_ms` Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY common_recv_time SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_cluster ( +`common_recv_time` Int64, + `common_direction` Int64, + `common_stream_dir` Int64, + `common_schema_type` String, + `common_client_ip` String, + `common_server_ip` String, + `common_client_port` Int64, + `common_server_port` Int64, + `common_app_label` String, + `common_l4_protocol` String, + `common_l7_protocol` String, + `common_isp` String, + `common_egress_link_id` Int64, + `common_ingress_link_id` Int64, + `http_host` String, + `http_url` String, + `http_cookie` String, + `http_referer` String, + `http_user_agent` String, + `http_request_line` String, + `http_response_line` String, + `http_status_code` Int64, + `ssl_sni` String, + `ssl_san` String, + `ssl_ja3_fingerprint` String, + `ssl_ja3_hash` String, + `ssl_cert_issuer` String, + `ssl_cert_subject` String, + `dns_qr` Int64, + `dns_opcode` Int64, + `dns_aa` Int64, + `dns_rcode` Int64, + `dns_qname` String, + `dns_qtype` Int64, + `dns_qclass` Int64, + `dns_sub` Int64, + `dns_rr` String, + `ssh_version` String, + `ssh_auth_success` String, + `ssh_client_version` String, + `ssh_server_version` String, + `ssh_cipher_alg` String, + `ssh_mac_alg` String, + `ssh_compression_alg` String, + `ssh_kex_alg` String, + `ssh_host_key_alg` String, + `ssh_host_key` String, + `ssh_hassh` String, + `egress_link_direction` String, + `ingress_link_direction` String, + `domain` String, + `domain_category_name` String, + `domain_category_group` String, + `domain_reputation_score` Int64, + `domain_reputation_level` String, + `domain_icp_company_name` String, + `domain_whois_org` String, + `client_country` String, + `client_province` String, + `client_region` String, + `client_longitude` Float64, + `client_latitude` Float64, + `client_isp` String, + `server_country` String, + `server_province` String, + `server_region` String, + `server_longitude` Float64, + `server_latitude` Float64, + `server_isp` String, + `app_category` String, + `app_subcategory` String, + `app_company` String, + `app_company_category` String, + `app_risk` Int64, + `dns_server_role` Array(String), + `dns_server_org` String, + `dns_server_os` String, + `dns_server_software` String, + `dns_protocol` String, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_lantency_ms` Int64, + `ssl_con_latency_ms` Int64, + `dns_response_latency_ms` Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_query ( +`common_recv_time` Int64, + `common_direction` Int64, + `common_stream_dir` Int64, + `common_schema_type` String, + `common_client_ip` String, + `common_server_ip` String, + `common_client_port` Int64, + `common_server_port` Int64, + `common_app_label` String, + `common_l4_protocol` String, + `common_l7_protocol` String, + `common_isp` String, + `common_egress_link_id` Int64, + `common_ingress_link_id` Int64, + `http_host` String, + `http_url` String, + `http_cookie` String, + `http_referer` String, + `http_user_agent` String, + `http_request_line` String, + `http_response_line` String, + `http_status_code` Int64, + `ssl_sni` String, + `ssl_san` String, + `ssl_ja3_fingerprint` String, + `ssl_ja3_hash` String, + `ssl_cert_issuer` String, + `ssl_cert_subject` String, + `dns_qr` Int64, + `dns_opcode` Int64, + `dns_aa` Int64, + `dns_rcode` Int64, + `dns_qname` String, + `dns_qtype` Int64, + `dns_qclass` Int64, + `dns_sub` Int64, + `dns_rr` String, + `ssh_version` String, + `ssh_auth_success` String, + `ssh_client_version` String, + `ssh_server_version` String, + `ssh_cipher_alg` String, + `ssh_mac_alg` String, + `ssh_compression_alg` String, + `ssh_kex_alg` String, + `ssh_host_key_alg` String, + `ssh_host_key` String, + `ssh_hassh` String, + `egress_link_direction` String, + `ingress_link_direction` String, + `domain` String, + `domain_category_name` String, + `domain_category_group` String, + `domain_reputation_score` Int64, + `domain_reputation_level` String, + `domain_icp_company_name` String, + `domain_whois_org` String, + `client_country` String, + `client_province` String, + `client_region` String, + `client_longitude` Float64, + `client_latitude` Float64, + `client_isp` String, + `server_country` String, + `server_province` String, + `server_region` String, + `server_longitude` Float64, + `server_latitude` Float64, + `server_isp` String, + `app_category` String, + `app_subcategory` String, + `app_company` String, + `app_company_category` String, + `app_risk` Int64, + `dns_server_role` Array(String), + `dns_server_org` String, + `dns_server_os` String, + `dns_server_software` String, + `dns_protocol` String, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_lantency_ms` Int64, + `ssl_con_latency_ms` Int64, + `dns_response_latency_ms` Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.security_event_local ON CLUSTER ck_cluster ( +`event_id` UInt64, + `start_time` Int64, + `event_severity` String, + `security_type` String, + `victim_ip` String, + `victim_location_country` String, + `victim_location_province` String, + `victim_location_region` String, + `victim_asn` String, + `offender_ip` String, + `offender_location_country` String, + `offender_location_province` String, + `offender_location_region` String, + `offender_asn` String, + `client_ip` String, + `server_ip` String, + `domain` String, + `domain_category_name` String, + `domain_category_group` String, + `domain_reputation_score` String, + `domain_reputation_level` String, + `app_name` String, + `app_category` String, + `app_subcategory` String, + `app_risk` Int64, + `malware_name` String, + `malware_alias` Array(String), + `malware_description` String, + `malware_platforms` Array(String), + `malware_techniques` Array(String), + `malware_groups` Array(String), + `phishing_url` String, + `phishing_brand_name` String, + `phishing_brand_industry` String, + `phishing_brand_website` String, + `phishing_language` String, + `cryptomining_pool` String, + `cryptomining_coin_type` String, + `cryptomining_software` String, + `dga_family` String, + `ddos_type` String, + `ddos_conditions` String, + `ddos_source_list` String +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(start_time)) ORDER BY (event_id, start_time) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.security_event ON CLUSTER ck_cluster ( +`event_id` UInt64, + `start_time` Int64, + `event_severity` String, + `security_type` String, + `victim_ip` String, + `victim_location_country` String, + `victim_location_province` String, + `victim_location_region` String, + `victim_asn` String, + `offender_ip` String, + `offender_location_country` String, + `offender_location_province` String, + `offender_location_region` String, + `offender_asn` String, + `client_ip` String, + `server_ip` String, + `domain` String, + `domain_category_name` String, + `domain_category_group` String, + `domain_reputation_score` String, + `domain_reputation_level` String, + `app_name` String, + `app_category` String, + `app_subcategory` String, + `app_risk` Int64, + `malware_name` String, + `malware_alias` Array(String), + `malware_description` String, + `malware_platforms` Array(String), + `malware_techniques` Array(String), + `malware_groups` Array(String), + `phishing_url` String, + `phishing_brand_name` String, + `phishing_brand_industry` String, + `phishing_brand_website` String, + `phishing_language` String, + `cryptomining_pool` String, + `cryptomining_coin_type` String, + `cryptomining_software` String, + `dga_family` String, + `ddos_type` String, + `ddos_conditions` String, + `ddos_source_list` String +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'security_event_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.security_event ON CLUSTER ck_query ( +`event_id` UInt64, + `start_time` Int64, + `event_severity` String, + `security_type` String, + `victim_ip` String, + `victim_location_country` String, + `victim_location_province` String, + `victim_location_region` String, + `victim_asn` String, + `offender_ip` String, + `offender_location_country` String, + `offender_location_province` String, + `offender_location_region` String, + `offender_asn` String, + `client_ip` String, + `server_ip` String, + `domain` String, + `domain_category_name` String, + `domain_category_group` String, + `domain_reputation_score` String, + `domain_reputation_level` String, + `app_name` String, + `app_category` String, + `app_subcategory` String, + `app_risk` Int64, + `malware_name` String, + `malware_alias` Array(String), + `malware_description` String, + `malware_platforms` Array(String), + `malware_techniques` Array(String), + `malware_groups` Array(String), + `phishing_url` String, + `phishing_brand_name` String, + `phishing_brand_industry` String, + `phishing_brand_website` String, + `phishing_language` String, + `cryptomining_pool` String, + `cryptomining_coin_type` String, + `cryptomining_software` String, + `dga_family` String, + `ddos_type` String, + `ddos_conditions` String, + `ddos_source_list` String +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'security_event_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_map_local ON CLUSTER ck_cluster ( +`server_country` String, + `client_country` String, + `server_province` String, + `client_province` String, + `server_region` String, + `client_region` String, + `common_recv_time` Int64, + `common_schema_type` String, + `common_l4_protocol` String, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_latency_ms` Int64, + `ssl_con_latency_ms` Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_recv_time, server_country, common_l4_protocol, common_schema_type, server_region) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_map ON CLUSTER ck_cluster ( +`server_country` String, + `client_country` String, + `server_province` String, + `client_province` String, + `server_region` String, + `client_region` String, + `common_recv_time` Int64, + `common_schema_type` String, + `common_l4_protocol` String, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_latency_ms` Int64, + `ssl_con_latency_ms` Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_map_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_map ON CLUSTER ck_query ( +`server_country` String, + `client_country` String, + `server_province` String, + `client_province` String, + `server_region` String, + `client_region` String, + `common_recv_time` Int64, + `common_schema_type` String, + `common_l4_protocol` String, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_latency_ms` Int64, + `ssl_con_latency_ms` Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_map_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_top_entity_local ON CLUSTER ck_cluster ( +`common_client_ip` String, + `common_server_ip` String, + `domain` String, + `common_app_label` LowCardinality(String), + `common_recv_time` Int64, + `group_by` LowCardinality(String), + `order_by` LowCardinality(String), + `time_granularity` Int64, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_latency_ms` Int64, + `ssl_con_latency_ms` Int64, + `packet_loss` Float64, + `packet_retrans` Float64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (group_by, order_by, common_recv_time, time_granularity) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_top_entity ON CLUSTER ck_cluster ( +`common_client_ip` String, + `common_server_ip` String, + `domain` String, + `common_app_label` LowCardinality(String), + `common_recv_time` Int64, + `group_by` LowCardinality(String), + `order_by` LowCardinality(String), + `time_granularity` Int64, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_latency_ms` Int64, + `ssl_con_latency_ms` Int64, + `packet_loss` Float64, + `packet_retrans` Float64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_top_entity_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_top_entity ON CLUSTER ck_query ( +`common_client_ip` String, + `common_server_ip` String, + `domain` String, + `common_app_label` LowCardinality(String), + `common_recv_time` Int64, + `group_by` LowCardinality(String), + `order_by` LowCardinality(String), + `time_granularity` Int64, + `common_c2s_pkt_num` Int64, + `common_c2s_byte_num` Int64, + `common_s2c_pkt_num` Int64, + `common_s2c_byte_num` Int64, + `common_sessions` Int64, + `common_c2s_tcp_lostlen` Int64, + `common_s2c_tcp_lostlen` Int64, + `common_c2s_tcp_unorder_num` Int64, + `common_s2c_tcp_unorder_num` Int64, + `common_c2s_byte_retrans` Int64, + `common_s2c_byte_retrans` Int64, + `common_c2s_pkt_retrans` Int64, + `common_s2c_pkt_retrans` Int64, + `common_establish_latency_ms` Int64, + `http_response_latency_ms` Int64, + `ssl_con_latency_ms` Int64, + `packet_loss` Float64, + `packet_retrans` Float64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_top_entity_local', rand()); +