gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CN 23.12 update CK session_record_cn表重构

Browse Source
This commit is contained in:
lifengchao
2023-12-04 10:06:53 +08:00
parent ad76a5c323
commit f5989f0375
2 changed files with 675 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

342
CN发布版本更新记录/CN-23.12/clickhouse/update-23.12-ck.sql
View File

@@ -657,3 +657,345 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE
avg_http_response_latency_ms Nullable(Float64), avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64) avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand()); ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUSTER ck_cluster (
recv_time Int64,
log_id UInt64,
flags UInt64,
start_timestamp_ms Datetime64(3),
end_timestamp_ms Datetime64(3),
duration_ms Int64,
decoded_as String,
client_ip String,
server_ip String,
client_port Int64,
server_port Int64,
app String,
app_path String,
protocol_path String,
l7_protocol String,
out_link_id Nullable(Int64),
in_link_id Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
http_host String,
http_url String,
http_cookie String,
http_referer String,
http_user_agent String,
http_request_line String,
http_response_line String,
http_status_code Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
dns_qr Nullable(Int64),
dns_opcode Nullable(Int64),
dns_aa Nullable(Int64),
dns_rcode Nullable(Int64),
dns_qname String,
dns_qtype Nullable(Int64),
dns_qclass Nullable(Int64),
dns_sub Nullable(Int64),
dns_rr String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
out_link_direction String,
in_link_direction String,
domain String,
domain_sld String,
domain_category_name String,
domain_category_group String,
domain_reputation_level String,
domain_icp_company_name String,
domain_whois_org String,
domain_tags Array(String),
client_zone String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
client_isp String,
client_asn String,
client_ip_tags Array(String),
server_zone String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
server_isp String,
server_asn String,
server_ip_tags Array(String),
app_category String,
app_subcategory String,
app_company String,
app_company_category String,
app_tags Array(String),
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64 DEFAULT 1,
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_rtt_ms Nullable(Int64),
http_response_latency_ms Nullable(Int64),
ssl_handshake_latency_ms Nullable(Int64),
dns_response_latency_ms Nullable(Int64)
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY common_recv_time SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_cluster (
recv_time Int64,
log_id UInt64,
flags UInt64,
start_timestamp_ms Datetime64(3),
end_timestamp_ms Datetime64(3),
duration_ms Int64,
decoded_as String,
client_ip String,
server_ip String,
client_port Int64,
server_port Int64,
app String,
app_path String,
protocol_path String,
l7_protocol String,
out_link_id Nullable(Int64),
in_link_id Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
http_host String,
http_url String,
http_cookie String,
http_referer String,
http_user_agent String,
http_request_line String,
http_response_line String,
http_status_code Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
dns_qr Nullable(Int64),
dns_opcode Nullable(Int64),
dns_aa Nullable(Int64),
dns_rcode Nullable(Int64),
dns_qname String,
dns_qtype Nullable(Int64),
dns_qclass Nullable(Int64),
dns_sub Nullable(Int64),
dns_rr String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
out_link_direction String,
in_link_direction String,
domain String,
domain_sld String,
domain_category_name String,
domain_category_group String,
domain_reputation_level String,
domain_icp_company_name String,
domain_whois_org String,
domain_tags Array(String),
client_zone String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
client_isp String,
client_asn String,
client_ip_tags Array(String),
server_zone String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
server_isp String,
server_asn String,
server_ip_tags Array(String),
app_category String,
app_subcategory String,
app_company String,
app_company_category String,
app_tags Array(String),
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64 DEFAULT 1,
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_rtt_ms Nullable(Int64),
http_response_latency_ms Nullable(Int64),
ssl_handshake_latency_ms Nullable(Int64),
dns_response_latency_ms Nullable(Int64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_query (
recv_time Int64,
log_id UInt64,
flags UInt64,
start_timestamp_ms Datetime64(3),
end_timestamp_ms Datetime64(3),
duration_ms Int64,
decoded_as String,
client_ip String,
server_ip String,
client_port Int64,
server_port Int64,
app String,
app_path String,
protocol_path String,
l7_protocol String,
out_link_id Nullable(Int64),
in_link_id Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
http_host String,
http_url String,
http_cookie String,
http_referer String,
http_user_agent String,
http_request_line String,
http_response_line String,
http_status_code Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
dns_qr Nullable(Int64),
dns_opcode Nullable(Int64),
dns_aa Nullable(Int64),
dns_rcode Nullable(Int64),
dns_qname String,
dns_qtype Nullable(Int64),
dns_qclass Nullable(Int64),
dns_sub Nullable(Int64),
dns_rr String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
out_link_direction String,
in_link_direction String,
domain String,
domain_sld String,
domain_category_name String,
domain_category_group String,
domain_reputation_level String,
domain_icp_company_name String,
domain_whois_org String,
domain_tags Array(String),
client_zone String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
client_isp String,
client_asn String,
client_ip_tags Array(String),
server_zone String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
server_isp String,
server_asn String,
server_ip_tags Array(String),
app_category String,
app_subcategory String,
app_company String,
app_company_category String,
app_tags Array(String),
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64 DEFAULT 1,
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_rtt_ms Nullable(Int64),
http_response_latency_ms Nullable(Int64),
ssl_handshake_latency_ms Nullable(Int64),
dns_response_latency_ms Nullable(Int64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand());

217
Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql
View File

@@ -3,23 +3,28 @@ create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_query;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUSTER ck_cluster ( CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUSTER ck_cluster (
common_recv_time Int64, recv_time Int64,
common_log_id UInt64, log_id UInt64,
common_flags UInt64, flags UInt64,
common_start_time Int64, start_timestamp_ms Datetime64(3),
common_end_time Int64, end_timestamp_ms Datetime64(3),
common_con_duration_ms Int64, duration_ms Int64,
common_schema_type String, decoded_as String,
common_client_ip String, client_ip String,
common_server_ip String, server_ip String,
common_client_port Int64, client_port Int64,
common_server_port Int64, server_port Int64,
common_app_label String, app String,
common_app_full_path String, app_path String,
common_l4_protocol String, protocol_path String,
common_l7_protocol String, l7_protocol String,
common_out_link_id Nullable(Int64), out_link_id Nullable(Int64),
common_in_link_id Nullable(Int64), in_link_id Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
http_host String, http_host String,
http_url String, http_url String,
http_cookie String, http_cookie String,
@@ -28,8 +33,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUS
http_request_line String, http_request_line String,
http_response_line String, http_response_line String,
http_status_code Nullable(Int64), http_status_code Nullable(Int64),
ssl_sni String,
ssl_version String, ssl_version String,
ssl_sni String,
ssl_san String, ssl_san String,
ssl_ja3_hash String, ssl_ja3_hash String,
ssl_ja3s_hash String, ssl_ja3s_hash String,
@@ -58,6 +63,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUS
stratum_cryptocurrency String, stratum_cryptocurrency String,
stratum_mining_pools String, stratum_mining_pools String,
stratum_mining_program String, stratum_mining_program String,
stratum_mining_subscribe String,
out_link_direction String, out_link_direction String,
in_link_direction String, in_link_direction String,
domain String, domain String,
@@ -91,43 +97,48 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUS
app_company String, app_company String,
app_company_category String, app_company_category String,
app_tags Array(String), app_tags Array(String),
common_c2s_pkt_num Int64, sent_pkts Int64,
common_c2s_byte_num Int64, sent_bytes Int64,
common_s2c_pkt_num Int64, received_pkts Int64,
common_s2c_byte_num Int64, received_bytes Int64,
common_sessions Int64, sessions Int64 DEFAULT 1,
common_c2s_tcp_lostlen Nullable(Int64), tcp_c2s_lost_bytes Nullable(Int64),
common_s2c_tcp_lostlen Nullable(Int64), tcp_s2c_lost_bytes Nullable(Int64),
common_c2s_tcp_unorder_num Nullable(Int64), tcp_c2s_o3_pkts Nullable(Int64),
common_s2c_tcp_unorder_num Nullable(Int64), tcp_s2c_o3_pkts Nullable(Int64),
common_c2s_byte_retrans Nullable(Int64), tcp_c2s_rtx_bytes Nullable(Int64),
common_s2c_byte_retrans Nullable(Int64), tcp_s2c_rtx_bytes Nullable(Int64),
common_c2s_pkt_retrans Nullable(Int64), tcp_c2s_rtx_pkts Nullable(Int64),
common_s2c_pkt_retrans Nullable(Int64), tcp_s2c_rtx_pkts Nullable(Int64),
common_establish_latency_ms Nullable(Int64), tcp_rtt_ms Nullable(Int64),
http_response_latency_ms Nullable(Int64), http_response_latency_ms Nullable(Int64),
ssl_con_latency_ms Nullable(Int64), ssl_handshake_latency_ms Nullable(Int64),
dns_response_latency_ms Nullable(Int64) dns_response_latency_ms Nullable(Int64)
) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY common_recv_time SETTINGS index_granularity = 8192; ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY common_recv_time SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_cluster ( CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_cluster (
common_recv_time Int64, recv_time Int64,
common_log_id UInt64, log_id UInt64,
common_flags UInt64, flags UInt64,
common_start_time Int64, start_timestamp_ms Datetime64(3),
common_end_time Int64, end_timestamp_ms Datetime64(3),
common_con_duration_ms Int64, duration_ms Int64,
common_schema_type String, decoded_as String,
common_client_ip String, client_ip String,
common_server_ip String, server_ip String,
common_client_port Int64, client_port Int64,
common_server_port Int64, server_port Int64,
common_app_label String, app String,
common_app_full_path String, app_path String,
common_l4_protocol String, protocol_path String,
common_l7_protocol String, l7_protocol String,
common_out_link_id Nullable(Int64), out_link_id Nullable(Int64),
common_in_link_id Nullable(Int64), in_link_id Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
http_host String, http_host String,
http_url String, http_url String,
http_cookie String, http_cookie String,
@@ -136,8 +147,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck
http_request_line String, http_request_line String,
http_response_line String, http_response_line String,
http_status_code Nullable(Int64), http_status_code Nullable(Int64),
ssl_sni String,
ssl_version String, ssl_version String,
ssl_sni String,
ssl_san String, ssl_san String,
ssl_ja3_hash String, ssl_ja3_hash String,
ssl_ja3s_hash String, ssl_ja3s_hash String,
@@ -166,6 +177,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck
stratum_cryptocurrency String, stratum_cryptocurrency String,
stratum_mining_pools String, stratum_mining_pools String,
stratum_mining_program String, stratum_mining_program String,
stratum_mining_subscribe String,
out_link_direction String, out_link_direction String,
in_link_direction String, in_link_direction String,
domain String, domain String,
@@ -199,44 +211,48 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck
app_company String, app_company String,
app_company_category String, app_company_category String,
app_tags Array(String), app_tags Array(String),
sent_pkts Int64,
common_c2s_pkt_num Int64, sent_bytes Int64,
common_c2s_byte_num Int64, received_pkts Int64,
common_s2c_pkt_num Int64, received_bytes Int64,
common_s2c_byte_num Int64, sessions Int64 DEFAULT 1,
common_sessions Int64, tcp_c2s_lost_bytes Nullable(Int64),
common_c2s_tcp_lostlen Nullable(Int64), tcp_s2c_lost_bytes Nullable(Int64),
common_s2c_tcp_lostlen Nullable(Int64), tcp_c2s_o3_pkts Nullable(Int64),
common_c2s_tcp_unorder_num Nullable(Int64), tcp_s2c_o3_pkts Nullable(Int64),
common_s2c_tcp_unorder_num Nullable(Int64), tcp_c2s_rtx_bytes Nullable(Int64),
common_c2s_byte_retrans Nullable(Int64), tcp_s2c_rtx_bytes Nullable(Int64),
common_s2c_byte_retrans Nullable(Int64), tcp_c2s_rtx_pkts Nullable(Int64),
common_c2s_pkt_retrans Nullable(Int64), tcp_s2c_rtx_pkts Nullable(Int64),
common_s2c_pkt_retrans Nullable(Int64), tcp_rtt_ms Nullable(Int64),
common_establish_latency_ms Nullable(Int64),
http_response_latency_ms Nullable(Int64), http_response_latency_ms Nullable(Int64),
ssl_con_latency_ms Nullable(Int64), ssl_handshake_latency_ms Nullable(Int64),
dns_response_latency_ms Nullable(Int64) dns_response_latency_ms Nullable(Int64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand()); ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_query ( CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_query (
common_recv_time Int64, recv_time Int64,
common_log_id UInt64, log_id UInt64,
common_flags UInt64, flags UInt64,
common_start_time Int64, start_timestamp_ms Datetime64(3),
common_end_time Int64, end_timestamp_ms Datetime64(3),
common_con_duration_ms Int64, duration_ms Int64,
common_schema_type String, decoded_as String,
common_client_ip String, client_ip String,
common_server_ip String, server_ip String,
common_client_port Int64, client_port Int64,
common_server_port Int64, server_port Int64,
common_app_label String, app String,
common_app_full_path String, app_path String,
common_l4_protocol String, protocol_path String,
common_l7_protocol String, l7_protocol String,
common_out_link_id Nullable(Int64), out_link_id Nullable(Int64),
common_in_link_id Nullable(Int64), in_link_id Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
http_host String, http_host String,
http_url String, http_url String,
http_cookie String, http_cookie String,
@@ -245,8 +261,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck
http_request_line String, http_request_line String,
http_response_line String, http_response_line String,
http_status_code Nullable(Int64), http_status_code Nullable(Int64),
ssl_sni String,
ssl_version String, ssl_version String,
ssl_sni String,
ssl_san String, ssl_san String,
ssl_ja3_hash String, ssl_ja3_hash String,
ssl_ja3s_hash String, ssl_ja3s_hash String,
@@ -275,6 +291,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck
stratum_cryptocurrency String, stratum_cryptocurrency String,
stratum_mining_pools String, stratum_mining_pools String,
stratum_mining_program String, stratum_mining_program String,
stratum_mining_subscribe String,
out_link_direction String, out_link_direction String,
in_link_direction String, in_link_direction String,
domain String, domain String,
@@ -308,22 +325,22 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck
app_company String, app_company String,
app_company_category String, app_company_category String,
app_tags Array(String), app_tags Array(String),
common_c2s_pkt_num Int64, sent_pkts Int64,
common_c2s_byte_num Int64, sent_bytes Int64,
common_s2c_pkt_num Int64, received_pkts Int64,
common_s2c_byte_num Int64, received_bytes Int64,
common_sessions Int64, sessions Int64 DEFAULT 1,
common_c2s_tcp_lostlen Nullable(Int64), tcp_c2s_lost_bytes Nullable(Int64),
common_s2c_tcp_lostlen Nullable(Int64), tcp_s2c_lost_bytes Nullable(Int64),
common_c2s_tcp_unorder_num Nullable(Int64), tcp_c2s_o3_pkts Nullable(Int64),
common_s2c_tcp_unorder_num Nullable(Int64), tcp_s2c_o3_pkts Nullable(Int64),
common_c2s_byte_retrans Nullable(Int64), tcp_c2s_rtx_bytes Nullable(Int64),
common_s2c_byte_retrans Nullable(Int64), tcp_s2c_rtx_bytes Nullable(Int64),
common_c2s_pkt_retrans Nullable(Int64), tcp_c2s_rtx_pkts Nullable(Int64),
common_s2c_pkt_retrans Nullable(Int64), tcp_s2c_rtx_pkts Nullable(Int64),
common_establish_latency_ms Nullable(Int64), tcp_rtt_ms Nullable(Int64),
http_response_latency_ms Nullable(Int64), http_response_latency_ms Nullable(Int64),
ssl_con_latency_ms Nullable(Int64), ssl_handshake_latency_ms Nullable(Int64),
dns_response_latency_ms Nullable(Int64) dns_response_latency_ms Nullable(Int64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand()); ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand());