gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GAL-566 全量建表语句适配ck新部署模式，删除ck_query,ck_all集群

Browse Source
This commit is contained in:
wangkuan
2024-05-22 15:32:38 +08:00
parent 53add1c5ea
commit e1aa09eaec
13 changed files with 7875 additions and 743 deletions
Download Patch File Download Diff File Expand all files Collapse all files

642
cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql
View File

@@ -1,5 +1,4 @@
create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_cluster;
create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_query;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn_local ON CLUSTER ck_cluster (
@@ -232,120 +231,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck
dns_response_latency_ms Nullable(Int64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_cn ON CLUSTER ck_query (
recv_time Int64,
log_id UInt64,
flags UInt64,
start_timestamp_ms DateTime64(3),
end_timestamp_ms DateTime64(3),
duration_ms Int64,
decoded_as String,
client_ip String,
server_ip String,
client_port Int64,
server_port Int64,
app String,
app_transition String,
decoded_path String,
ip_protocol LowCardinality(String),
l7_protocol String,
out_link_id Nullable(Int64),
in_link_id Nullable(Int64),
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
http_host String,
http_url String,
http_cookie String,
http_referer String,
http_user_agent String,
http_request_line String,
http_response_line String,
http_status_code Nullable(Int64),
ssl_version String,
ssl_sni String,
ssl_san String,
ssl_ja3_hash String,
ssl_ja3s_hash String,
ssl_cert_issuer String,
ssl_cert_subject String,
dns_qr Nullable(Int64),
dns_opcode Nullable(Int64),
dns_aa Nullable(Int64),
dns_rcode Nullable(Int64),
dns_qname String,
dns_qtype Nullable(Int64),
dns_qclass Nullable(Int64),
dns_sub Nullable(Int64),
dns_rr String,
ssh_version String,
ssh_auth_success String,
ssh_client_version String,
ssh_server_version String,
ssh_cipher_alg String,
ssh_mac_alg String,
ssh_compression_alg String,
ssh_kex_alg String,
ssh_host_key_alg String,
ssh_host_key String,
ssh_hassh String,
stratum_cryptocurrency String,
stratum_mining_pools String,
stratum_mining_program String,
stratum_mining_subscribe String,
out_link_direction String,
in_link_direction String,
domain String,
domain_sld String,
domain_category_name String,
domain_category_group String,
domain_reputation_level String,
domain_icp_company_name String,
domain_whois_org String,
domain_tags Array(String),
client_zone String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
client_isp String,
client_asn String,
client_ip_tags Array(String),
server_zone String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
server_isp String,
server_asn String,
server_ip_tags Array(String),
app_category String,
app_subcategory String,
app_company String,
app_company_category String,
app_tags Array(String),
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64 DEFAULT 1,
tcp_c2s_lost_bytes Nullable(Int64),
tcp_s2c_lost_bytes Nullable(Int64),
tcp_c2s_o3_pkts Nullable(Int64),
tcp_s2c_o3_pkts Nullable(Int64),
tcp_c2s_rtx_bytes Nullable(Int64),
tcp_s2c_rtx_bytes Nullable(Int64),
tcp_c2s_rtx_pkts Nullable(Int64),
tcp_s2c_rtx_pkts Nullable(Int64),
tcp_rtt_ms Nullable(Int64),
http_response_latency_ms Nullable(Int64),
ssl_handshake_latency_ms Nullable(Int64),
dns_response_latency_ms Nullable(Int64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_record_cn_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_local ON CLUSTER ck_cluster (
@@ -438,51 +323,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip ON CLUSTER ck_cluster
bidirectional_sessions Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_ip_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip ON CLUSTER ck_query (
ip String,
side String,
zone String,
stat_time Int64,
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64,
traffic_inbound_byte Int64,
traffic_inbound_pkt Int64,
traffic_outbound_byte Int64,
traffic_outbound_pkt Int64,
traffic_internal_byte Int64,
traffic_internal_pkt Int64,
traffic_through_byte Int64,
traffic_through_pkt Int64,
tcp_c2s_lost_bytes_ratio Nullable(Float64),
tcp_s2c_lost_bytes_ratio Nullable(Float64),
tcp_lost_bytes_ratio Nullable(Float64),
tcp_c2s_o3_pkts_ratio Nullable(Float64),
tcp_s2c_o3_pkts_ratio Nullable(Float64),
tcp_o3_pkts_ratio Nullable(Float64),
tcp_c2s_rtx_bytes_ratio Nullable(Float64),
tcp_s2c_rtx_bytes_ratio Nullable(Float64),
tcp_rtx_bytes_ratio Nullable(Float64),
tcp_c2s_rtx_pkts_ratio Nullable(Float64),
tcp_s2c_rtx_pkts_ratio Nullable(Float64),
tcp_rtx_pkts_ratio Nullable(Float64),
avg_tcp_rtt_ms Nullable(Float64),
avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64),
asymmetric_sessions Int64,
bulky_sessions Int64,
cbr_streaming_sessions Int64,
download_sessions Int64,
interactive_sessions Int64,
pseudo_unidirectional_sessions Int64,
streaming_sessions Int64,
unidirectional_sessions Int64,
random_looking_sessions Int64,
bidirectional_sessions Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_ip_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_region_local ON CLUSTER ck_cluster (
country_region String,
@@ -556,42 +396,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_region ON CLUSTER ck_clu
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_region_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_region ON CLUSTER ck_query (
country_region String,
super_admin_area String,
admin_area String,
side String,
stat_time Int64,
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64,
traffic_inbound_byte Int64,
traffic_inbound_pkt Int64,
traffic_outbound_byte Int64,
traffic_outbound_pkt Int64,
traffic_internal_byte Int64,
traffic_internal_pkt Int64,
traffic_through_byte Int64,
traffic_through_pkt Int64,
tcp_c2s_lost_bytes_ratio Nullable(Float64),
tcp_s2c_lost_bytes_ratio Nullable(Float64),
tcp_lost_bytes_ratio Nullable(Float64),
tcp_c2s_o3_pkts_ratio Nullable(Float64),
tcp_s2c_o3_pkts_ratio Nullable(Float64),
tcp_o3_pkts_ratio Nullable(Float64),
tcp_c2s_rtx_bytes_ratio Nullable(Float64),
tcp_s2c_rtx_bytes_ratio Nullable(Float64),
tcp_rtx_bytes_ratio Nullable(Float64),
tcp_c2s_rtx_pkts_ratio Nullable(Float64),
tcp_s2c_rtx_pkts_ratio Nullable(Float64),
tcp_rtx_pkts_ratio Nullable(Float64),
avg_tcp_rtt_ms Nullable(Float64),
avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_region_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_asn_local ON CLUSTER ck_cluster (
asn String,
@@ -663,40 +467,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_asn ON CLUSTER ck_cluste
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_asn_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_asn ON CLUSTER ck_query (
asn String,
isp String,
side String,
stat_time Int64,
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64,
traffic_inbound_byte Int64,
traffic_inbound_pkt Int64,
traffic_outbound_byte Int64,
traffic_outbound_pkt Int64,
traffic_internal_byte Int64,
traffic_internal_pkt Int64,
traffic_through_byte Int64,
traffic_through_pkt Int64,
tcp_c2s_lost_bytes_ratio Nullable(Float64),
tcp_s2c_lost_bytes_ratio Nullable(Float64),
tcp_lost_bytes_ratio Nullable(Float64),
tcp_c2s_o3_pkts_ratio Nullable(Float64),
tcp_s2c_o3_pkts_ratio Nullable(Float64),
tcp_o3_pkts_ratio Nullable(Float64),
tcp_c2s_rtx_bytes_ratio Nullable(Float64),
tcp_s2c_rtx_bytes_ratio Nullable(Float64),
tcp_rtx_bytes_ratio Nullable(Float64),
tcp_c2s_rtx_pkts_ratio Nullable(Float64),
tcp_s2c_rtx_pkts_ratio Nullable(Float64),
tcp_rtx_pkts_ratio Nullable(Float64),
avg_tcp_rtt_ms Nullable(Float64),
avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_asn_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_application_local ON CLUSTER ck_cluster (
@@ -772,41 +542,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_application ON CLUSTER c
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_application_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_application ON CLUSTER ck_query (
app String,
app_category String,
app_subcategory String,
app_company String,
stat_time Int64,
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64,
traffic_inbound_byte Int64,
traffic_inbound_pkt Int64,
traffic_outbound_byte Int64,
traffic_outbound_pkt Int64,
traffic_internal_byte Int64,
traffic_internal_pkt Int64,
traffic_through_byte Int64,
traffic_through_pkt Int64,
tcp_c2s_lost_bytes_ratio Nullable(Float64),
tcp_s2c_lost_bytes_ratio Nullable(Float64),
tcp_lost_bytes_ratio Nullable(Float64),
tcp_c2s_o3_pkts_ratio Nullable(Float64),
tcp_s2c_o3_pkts_ratio Nullable(Float64),
tcp_o3_pkts_ratio Nullable(Float64),
tcp_c2s_rtx_bytes_ratio Nullable(Float64),
tcp_s2c_rtx_bytes_ratio Nullable(Float64),
tcp_rtx_bytes_ratio Nullable(Float64),
tcp_c2s_rtx_pkts_ratio Nullable(Float64),
tcp_s2c_rtx_pkts_ratio Nullable(Float64),
tcp_rtx_pkts_ratio Nullable(Float64),
avg_tcp_rtt_ms Nullable(Float64),
avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_application_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_domain_local ON CLUSTER ck_cluster (
@@ -881,41 +616,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_domain ON CLUSTER ck_clu
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_domain_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_domain ON CLUSTER ck_query (
domain String,
domain_sld String,
domain_category_name String,
domain_category_group String,
stat_time Int64,
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64,
traffic_inbound_byte Int64,
traffic_inbound_pkt Int64,
traffic_outbound_byte Int64,
traffic_outbound_pkt Int64,
traffic_internal_byte Int64,
traffic_internal_pkt Int64,
traffic_through_byte Int64,
traffic_through_pkt Int64,
tcp_c2s_lost_bytes_ratio Nullable(Float64),
tcp_s2c_lost_bytes_ratio Nullable(Float64),
tcp_lost_bytes_ratio Nullable(Float64),
tcp_c2s_o3_pkts_ratio Nullable(Float64),
tcp_s2c_o3_pkts_ratio Nullable(Float64),
tcp_o3_pkts_ratio Nullable(Float64),
tcp_c2s_rtx_bytes_ratio Nullable(Float64),
tcp_s2c_rtx_bytes_ratio Nullable(Float64),
tcp_rtx_bytes_ratio Nullable(Float64),
tcp_c2s_rtx_pkts_ratio Nullable(Float64),
tcp_s2c_rtx_pkts_ratio Nullable(Float64),
tcp_rtx_pkts_ratio Nullable(Float64),
avg_tcp_rtt_ms Nullable(Float64),
avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_domain_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_protocol_local ON CLUSTER ck_cluster (
@@ -986,39 +686,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_protocol ON CLUSTER ck_c
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_protocol_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_protocol ON CLUSTER ck_query (
l7_protocol String,
server_port Int64,
stat_time Int64,
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64,
traffic_inbound_byte Int64,
traffic_inbound_pkt Int64,
traffic_outbound_byte Int64,
traffic_outbound_pkt Int64,
traffic_internal_byte Int64,
traffic_internal_pkt Int64,
traffic_through_byte Int64,
traffic_through_pkt Int64,
tcp_c2s_lost_bytes_ratio Nullable(Float64),
tcp_s2c_lost_bytes_ratio Nullable(Float64),
tcp_lost_bytes_ratio Nullable(Float64),
tcp_c2s_o3_pkts_ratio Nullable(Float64),
tcp_s2c_o3_pkts_ratio Nullable(Float64),
tcp_o3_pkts_ratio Nullable(Float64),
tcp_c2s_rtx_bytes_ratio Nullable(Float64),
tcp_s2c_rtx_bytes_ratio Nullable(Float64),
tcp_rtx_bytes_ratio Nullable(Float64),
tcp_c2s_rtx_pkts_ratio Nullable(Float64),
tcp_s2c_rtx_pkts_ratio Nullable(Float64),
tcp_rtx_pkts_ratio Nullable(Float64),
avg_tcp_rtt_ms Nullable(Float64),
avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_protocol_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_link_local ON CLUSTER ck_cluster (
@@ -1109,50 +776,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_link ON CLUSTER ck_clust
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_link_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_link ON CLUSTER ck_query (
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_zone String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_zone String,
out_link_id Int64,
in_link_id Int64,
out_link_direction String,
in_link_direction String,
stat_time Int64,
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64,
traffic_inbound_byte Int64,
traffic_inbound_pkt Int64,
traffic_outbound_byte Int64,
traffic_outbound_pkt Int64,
traffic_internal_byte Int64,
traffic_internal_pkt Int64,
traffic_through_byte Int64,
traffic_through_pkt Int64,
tcp_c2s_lost_bytes_ratio Nullable(Float64),
tcp_s2c_lost_bytes_ratio Nullable(Float64),
tcp_lost_bytes_ratio Nullable(Float64),
tcp_c2s_o3_pkts_ratio Nullable(Float64),
tcp_s2c_o3_pkts_ratio Nullable(Float64),
tcp_o3_pkts_ratio Nullable(Float64),
tcp_c2s_rtx_bytes_ratio Nullable(Float64),
tcp_s2c_rtx_bytes_ratio Nullable(Float64),
tcp_rtx_bytes_ratio Nullable(Float64),
tcp_c2s_rtx_pkts_ratio Nullable(Float64),
tcp_s2c_rtx_pkts_ratio Nullable(Float64),
tcp_rtx_pkts_ratio Nullable(Float64),
avg_tcp_rtt_ms Nullable(Float64),
avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_link_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app_local ON CLUSTER ck_cluster (
subscriber_id String,
@@ -1230,43 +853,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTER ck_query (
subscriber_id String,
app String,
imei String,
imsi String,
phone_number String,
apn String,
stat_time Int64,
sent_pkts Int64,
sent_bytes Int64,
received_pkts Int64,
received_bytes Int64,
sessions Int64,
traffic_inbound_byte Int64,
traffic_inbound_pkt Int64,
traffic_outbound_byte Int64,
traffic_outbound_pkt Int64,
traffic_internal_byte Int64,
traffic_internal_pkt Int64,
traffic_through_byte Int64,
traffic_through_pkt Int64,
tcp_c2s_lost_bytes_ratio Nullable(Float64),
tcp_s2c_lost_bytes_ratio Nullable(Float64),
tcp_lost_bytes_ratio Nullable(Float64),
tcp_c2s_o3_pkts_ratio Nullable(Float64),
tcp_s2c_o3_pkts_ratio Nullable(Float64),
tcp_o3_pkts_ratio Nullable(Float64),
tcp_c2s_rtx_bytes_ratio Nullable(Float64),
tcp_s2c_rtx_bytes_ratio Nullable(Float64),
tcp_rtx_bytes_ratio Nullable(Float64),
tcp_c2s_rtx_pkts_ratio Nullable(Float64),
tcp_s2c_rtx_pkts_ratio Nullable(Float64),
tcp_rtx_pkts_ratio Nullable(Float64),
avg_tcp_rtt_ms Nullable(Float64),
avg_http_response_latency_ms Nullable(Float64),
avg_ssl_handshake_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand());
CREATE TABLE cyber_narrator_galaxy.metric_tag_local
@@ -1291,14 +877,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_cluste
domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_query (
tag String,
stat_time Int64,
ip_sketch String,
domain_sketch String,
ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster (
@@ -1331,20 +909,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_query (
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,
second_location String,
third_location String,
data_source String,
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_server_ip_local ON CLUSTER ck_cluster (
@@ -1379,21 +943,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_server_ip ON CLUSTER
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_server_ip_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_server_ip ON CLUSTER ck_query (
server_ip String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_isp String,
server_role Array(String),
stat_time Int64,
query_num Int64,
traffic_inbound_byte Int64,
traffic_outbound_byte Int64,
internal_query_num Int64,
external_query_num Int64,
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_server_ip_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qname_local ON CLUSTER ck_cluster (
@@ -1422,18 +971,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qname ON CLUSTER ck_
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_qname_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qname ON CLUSTER ck_query (
qname String,
qname_sld String,
qname_tld String,
stat_time Int64,
query_num Int64,
traffic_inbound_byte Int64,
traffic_outbound_byte Int64,
internal_query_num Int64,
external_query_num Int64,
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_qname_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qtype_local ON CLUSTER ck_cluster (
@@ -1458,17 +995,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qtype ON CLUSTER ck_
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_qtype_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_qtype ON CLUSTER ck_query (
qtype Int64,
stat_time Int64,
query_num Int64,
traffic_inbound_byte Int64,
traffic_outbound_byte Int64,
internal_query_num Int64,
external_query_num Int64,
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_qtype_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rcode_local ON CLUSTER ck_cluster (
rcode Int64,
@@ -1492,17 +1018,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rcode ON CLUSTER ck_
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rcode_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rcode ON CLUSTER ck_query (
rcode Int64,
stat_time Int64,
query_num Int64,
traffic_inbound_byte Int64,
traffic_outbound_byte Int64,
internal_query_num Int64,
external_query_num Int64,
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rcode_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_a_local ON CLUSTER ck_cluster (
rr_a String,
@@ -1526,17 +1041,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_a ON CLUSTER ck_c
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_a_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_a ON CLUSTER ck_query (
rr_a String,
stat_time Int64,
query_num Int64,
traffic_inbound_byte Int64,
traffic_outbound_byte Int64,
internal_query_num Int64,
external_query_num Int64,
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_a_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_aaaa_local ON CLUSTER ck_cluster (
rr_aaaa String,
@@ -1560,18 +1064,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_aaaa ON CLUSTER c
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_aaaa_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_aaaa ON CLUSTER ck_query (
rr_aaaa String,
stat_time Int64,
query_num Int64,
traffic_inbound_byte Int64,
traffic_outbound_byte Int64,
internal_query_num Int64,
external_query_num Int64,
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_aaaa_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_cname_local ON CLUSTER ck_cluster (
rr_cname String,
stat_time Int64,
@@ -1594,18 +1086,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_cname ON CLUSTER
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_cname_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_cname ON CLUSTER ck_query (
rr_cname String,
stat_time Int64,
query_num Int64,
traffic_inbound_byte Int64,
traffic_outbound_byte Int64,
internal_query_num Int64,
external_query_num Int64,
avg_response_latency_ms Nullable(Float64)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_cname_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local ON CLUSTER ck_cluster (
app_name String,
domain String,
@@ -1640,23 +1120,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_query (
app_name String,
domain String,
ip String,
ip_country_region String,
ip_super_admin_area String,
ip_admin_area String,
ip_asn String,
ip_isp String,
domain_category_name String,
domain_category_group String,
app_category String,
app_subcategory String,
entity_tags Array(String),
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local ON CLUSTER ck_cluster (
ip String,
@@ -1672,14 +1135,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_query (
ip String,
l7_protocol String,
port Int64,
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app_local on cluster ck_cluster
(
app_name String,
@@ -1718,28 +1173,6 @@ TTL toDateTime(update_time) + toIntervalSecond(2592000),
app_subcategory = anyLast(app_subcategory),
entity_tags = groupUniqArrayArray(entity_tags);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_query
(
app_name String,
domain String,
ip String,
ip_country_region String,
ip_super_admin_area String,
ip_admin_area String,
ip_asn String,
ip_isp String,
domain_category_name String,
domain_category_group String,
app_category String,
app_subcategory String,
entity_tags Array(String),
create_time Int64,
update_time Int64
)
ENGINE = Distributed('ck_cluster',
'cyber_narrator_galaxy',
'session_relation_domain_ip_app_local',
rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_cluster
(
@@ -1803,19 +1236,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app
update_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_query (
app_name String,
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
app_category String,
app_subcategory String,
create_time Int64,
update_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app_local ON CLUSTER ck_cluster (
app_name String,
@@ -1841,19 +1261,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_query (
app_name String,
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
app_category String,
app_subcategory String,
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
CREATE TABLE if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_local on cluster ck_cluster
(
ip String,
@@ -1872,18 +1279,6 @@ TTL toDateTime(update_time) + toIntervalSecond(2592000),
l7_protocol SET create_time = min(create_time),
update_time = max(update_time);
CREATE TABLE if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute on cluster ck_query
(
ip String,
l7_protocol String,
port Int64,
create_time Int64,
update_time Int64
)
ENGINE = Distributed('ck_cluster',
'cyber_narrator_galaxy',
'cn_ip_dynamic_attribute_local',
rand());
CREATE TABLE if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute on cluster ck_cluster
(
@@ -1999,42 +1394,7 @@ ENGINE = Distributed('ck_cluster',
'cn_security_event_local',
rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_query
(
status Int64,
is_builtin Int64,
rule_type String,
victim_ip String,
victim_country_region String,
victim_super_admin_area String,
victim_admin_area String,
victim_longitude Nullable(Float64),
victim_latitude Nullable(Float64),
offender_ip String,
offender_country_region String,
offender_super_admin_area String,
offender_admin_area String,
offender_longitude Nullable(Float64),
offender_latitude Nullable(Float64),
event_info String,
event_key String,
severity String,
event_type String,
duration_s Int64,
event_name String,
app String,
domain String,
event_id Int64,
rule_id Int64,
start_time Int64,
end_time Int64,
match_times Int64
)
ENGINE = Distributed('ck_cluster',
'cyber_narrator_galaxy',
'cn_security_event_local',
rand());
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric on cluster ck_cluster
TO cyber_narrator_galaxy.session_relation_domain_ip_app_local

102
cyber_narrator/installation/clickhouse/cn_nfsp_clickhouse_ddl.sql
View File

@@ -1,5 +1,4 @@
create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_cluster;
create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_query;
@@ -208,104 +207,3 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_nfsp on cluster
)
ENGINE =Distributed(ck_cluster,cyber_narrator_galaxy,session_record_nfsp_local,rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_nfsp on cluster ck_query (
id Int32,
expiration_id Int32,
src_ip String,
src_mac String,
src_oui String,
src_port Int32,
dst_ip String,
dst_mac String,
dst_oui String,
dst_port Int32,
protocol Int32,
ip_version Int32,
vlan_id Int32,
bidirectional_first_seen_ms Int64,
bidirectional_last_seen_ms Int64,
bidirectional_duration_ms Int64,
bidirectional_packets Int64,
bidirectional_bytes Int64,
src2dst_first_seen_ms Int64,
src2dst_last_seen_ms Int64,
src2dst_duration_ms Int64,
src2dst_packets Int64,
src2dst_bytes Int64,
dst2src_first_seen_ms Int64,
dst2src_last_seen_ms Int64,
dst2src_duration_ms Int64,
dst2src_packets Int64,
dst2src_bytes Int64,
tunnel_id Int64,
application_name String,
application_category_name String,
application_is_guessed Int64,
application_confidence Int64,
requested_server_name String,
client_fingerprint String,
server_fingerprint String,
user_agent String,
content_type String,
bidirectional_min_ps Int64,
bidirectional_mean_ps Float64,
bidirectional_stddev_ps Float64,
bidirectional_max_ps Int64,
src2dst_min_ps Int64,
src2dst_mean_ps Float64,
src2dst_stddev_ps Float64,
src2dst_max_ps Int64,
dst2src_min_ps Int64,
dst2src_mean_ps Float64,
dst2src_stddev_ps Float64,
dst2src_max_ps Int64,
bidirectional_min_piat_ms Int64,
bidirectional_mean_piat_ms Float64,
bidirectional_stddev_piat_ms Float64,
bidirectional_max_piat_ms Int64,
src2dst_min_piat_ms Int64,
src2dst_mean_piat_ms Float64,
src2dst_stddev_piat_ms Float64,
src2dst_max_piat_ms Int64,
dst2src_min_piat_ms Int64,
dst2src_mean_piat_ms Float64,
dst2src_stddev_piat_ms Float64,
dst2src_max_piat_ms Int64,
bidirectional_syn_packets Int64,
bidirectional_cwr_packets Int64,
bidirectional_ece_packets Int64,
bidirectional_urg_packets Int64,
bidirectional_ack_packets Int64,
bidirectional_psh_packets Int64,
bidirectional_rst_packets Int64,
bidirectional_fin_packets Int64,
src2dst_syn_packets Int64,
src2dst_cwr_packets Int64,
src2dst_ece_packets Int64,
src2dst_urg_packets Int64,
src2dst_ack_packets Int64,
src2dst_psh_packets Int64,
src2dst_rst_packets Int64,
src2dst_fin_packets Int64,
dst2src_syn_packets Int64,
dst2src_cwr_packets Int64,
dst2src_ece_packets Int64,
dst2src_urg_packets Int64,
dst2src_ack_packets Int64,
dst2src_psh_packets Int64,
dst2src_rst_packets Int64,
dst2src_fin_packets Int64,
splt_direction Array(Int64),
splt_ps Array(Int64),
splt_piat_ms Array(Int64),
stf_payload_sizes Array(Int64),
stf_directions Array(Int64),
stf_gram_sequences Array(String),
stf_gram_match_results String,
stf_encrypted_tunnel_flag Int64,
stf_simple_obfs_resp_flag Int64,
stf_valid_packet_counts Int64,
stf_has_tcp_handshake Int64
)
ENGINE =Distributed(ck_cluster,cyber_narrator_galaxy,session_record_nfsp_local,rand());

2129
cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl.sql Normal file
View File

File diff suppressed because it is too large Load Diff

57
cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl_check.sql Normal file
View File

@@ -0,0 +1,57 @@
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_domain_ip_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, create_time, update_time
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT status, is_builtin, rule_type, victim_ip, offender_ip, event_info, event_key, severity, event_type, duration_s, event_name, app, `domain`, event_id, rule_id, start_time, end_time, match_times
FROM cyber_narrator_galaxy.cn_security_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name ,domain ,ip ,ip_country_region ,ip_super_admin_area ,ip_admin_area ,ip_asn ,ip_isp ,domain_category_name ,domain_category_group ,app_category ,app_subcategory ,entity_tags ,stat_time
FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name, subscriber_id, imei, imsi, phone_number, apn, app_category, app_subcategory, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_subscriber_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name , subscriber_id , imei , imsi , phone_number , apn , app_category , app_subcategory , stat_time
FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name,subscriber_id,imei,imsi,phone_number,apn,app_category,app_subcategory,create_time,update_time
FROM cyber_narrator_galaxy.session_relation_subscriber_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, create_time, update_time
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app, app_category, app_subcategory, app_company, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_application where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT asn, isp, side, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_asn where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT qname, qname_sld, qname_tld, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_qname where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT qtype, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_qtype where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rcode, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rcode where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_a, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_a where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_aaaa, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_aaaa where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_cname, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_cname where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT server_ip, server_country_region, server_super_admin_area, server_admin_area, server_isp, server_role, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_server_ip where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT `domain`, domain_sld, domain_category_name, domain_category_group, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_domain where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, side, `zone`, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms, asymmetric_sessions, bulky_sessions, cbr_streaming_sessions, download_sessions, interactive_sessions, pseudo_unidirectional_sessions, streaming_sessions, unidirectional_sessions, random_looking_sessions, bidirectional_sessions
FROM cyber_narrator_galaxy.metric_ip where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, stat_time
FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT client_country_region, client_super_admin_area, client_admin_area, client_zone, server_country_region, server_super_admin_area, server_admin_area, server_zone, out_link_id, in_link_id, out_link_direction, in_link_direction, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_link where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT l7_protocol, server_port, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_protocol where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT country_region, super_admin_area, admin_area, side, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_region where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state
FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

311
cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_nfsp_clickhouse_ddl.sql Normal file
View File

@@ -0,0 +1,311 @@
create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_cluster;
create database IF NOT EXISTS cyber_narrator_galaxy ON CLUSTER ck_query;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_nfsp_local on cluster ck_cluster (
id Int32,
expiration_id Int32,
src_ip String,
src_mac String,
src_oui String,
src_port Int32,
dst_ip String,
dst_mac String,
dst_oui String,
dst_port Int32,
protocol Int32,
ip_version Int32,
vlan_id Int32,
bidirectional_first_seen_ms Int64,
bidirectional_last_seen_ms Int64,
bidirectional_duration_ms Int64,
bidirectional_packets Int64,
bidirectional_bytes Int64,
src2dst_first_seen_ms Int64,
src2dst_last_seen_ms Int64,
src2dst_duration_ms Int64,
src2dst_packets Int64,
src2dst_bytes Int64,
dst2src_first_seen_ms Int64,
dst2src_last_seen_ms Int64,
dst2src_duration_ms Int64,
dst2src_packets Int64,
dst2src_bytes Int64,
tunnel_id Int64,
application_name String,
application_category_name String,
application_is_guessed Int64,
application_confidence Int64,
requested_server_name String,
client_fingerprint String,
server_fingerprint String,
user_agent String,
content_type String,
bidirectional_min_ps Int64,
bidirectional_mean_ps Float64,
bidirectional_stddev_ps Float64,
bidirectional_max_ps Int64,
src2dst_min_ps Int64,
src2dst_mean_ps Float64,
src2dst_stddev_ps Float64,
src2dst_max_ps Int64,
dst2src_min_ps Int64,
dst2src_mean_ps Float64,
dst2src_stddev_ps Float64,
dst2src_max_ps Int64,
bidirectional_min_piat_ms Int64,
bidirectional_mean_piat_ms Float64,
bidirectional_stddev_piat_ms Float64,
bidirectional_max_piat_ms Int64,
src2dst_min_piat_ms Int64,
src2dst_mean_piat_ms Float64,
src2dst_stddev_piat_ms Float64,
src2dst_max_piat_ms Int64,
dst2src_min_piat_ms Int64,
dst2src_mean_piat_ms Float64,
dst2src_stddev_piat_ms Float64,
dst2src_max_piat_ms Int64,
bidirectional_syn_packets Int64,
bidirectional_cwr_packets Int64,
bidirectional_ece_packets Int64,
bidirectional_urg_packets Int64,
bidirectional_ack_packets Int64,
bidirectional_psh_packets Int64,
bidirectional_rst_packets Int64,
bidirectional_fin_packets Int64,
src2dst_syn_packets Int64,
src2dst_cwr_packets Int64,
src2dst_ece_packets Int64,
src2dst_urg_packets Int64,
src2dst_ack_packets Int64,
src2dst_psh_packets Int64,
src2dst_rst_packets Int64,
src2dst_fin_packets Int64,
dst2src_syn_packets Int64,
dst2src_cwr_packets Int64,
dst2src_ece_packets Int64,
dst2src_urg_packets Int64,
dst2src_ack_packets Int64,
dst2src_psh_packets Int64,
dst2src_rst_packets Int64,
dst2src_fin_packets Int64,
splt_direction Array(Int64),
splt_ps Array(Int64),
splt_piat_ms Array(Int64),
stf_payload_sizes Array(Int64),
stf_directions Array(Int64),
stf_gram_sequences Array(String),
stf_gram_match_results String,
stf_encrypted_tunnel_flag Int64,
stf_simple_obfs_resp_flag Int64,
stf_valid_packet_counts Int64,
stf_has_tcp_handshake Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(bidirectional_first_seen_ms/1000))
ORDER BY bidirectional_first_seen_ms;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_nfsp on cluster ck_cluster (
id Int32,
expiration_id Int32,
src_ip String,
src_mac String,
src_oui String,
src_port Int32,
dst_ip String,
dst_mac String,
dst_oui String,
dst_port Int32,
protocol Int32,
ip_version Int32,
vlan_id Int32,
bidirectional_first_seen_ms Int64,
bidirectional_last_seen_ms Int64,
bidirectional_duration_ms Int64,
bidirectional_packets Int64,
bidirectional_bytes Int64,
src2dst_first_seen_ms Int64,
src2dst_last_seen_ms Int64,
src2dst_duration_ms Int64,
src2dst_packets Int64,
src2dst_bytes Int64,
dst2src_first_seen_ms Int64,
dst2src_last_seen_ms Int64,
dst2src_duration_ms Int64,
dst2src_packets Int64,
dst2src_bytes Int64,
tunnel_id Int64,
application_name String,
application_category_name String,
application_is_guessed Int64,
application_confidence Int64,
requested_server_name String,
client_fingerprint String,
server_fingerprint String,
user_agent String,
content_type String,
bidirectional_min_ps Int64,
bidirectional_mean_ps Float64,
bidirectional_stddev_ps Float64,
bidirectional_max_ps Int64,
src2dst_min_ps Int64,
src2dst_mean_ps Float64,
src2dst_stddev_ps Float64,
src2dst_max_ps Int64,
dst2src_min_ps Int64,
dst2src_mean_ps Float64,
dst2src_stddev_ps Float64,
dst2src_max_ps Int64,
bidirectional_min_piat_ms Int64,
bidirectional_mean_piat_ms Float64,
bidirectional_stddev_piat_ms Float64,
bidirectional_max_piat_ms Int64,
src2dst_min_piat_ms Int64,
src2dst_mean_piat_ms Float64,
src2dst_stddev_piat_ms Float64,
src2dst_max_piat_ms Int64,
dst2src_min_piat_ms Int64,
dst2src_mean_piat_ms Float64,
dst2src_stddev_piat_ms Float64,
dst2src_max_piat_ms Int64,
bidirectional_syn_packets Int64,
bidirectional_cwr_packets Int64,
bidirectional_ece_packets Int64,
bidirectional_urg_packets Int64,
bidirectional_ack_packets Int64,
bidirectional_psh_packets Int64,
bidirectional_rst_packets Int64,
bidirectional_fin_packets Int64,
src2dst_syn_packets Int64,
src2dst_cwr_packets Int64,
src2dst_ece_packets Int64,
src2dst_urg_packets Int64,
src2dst_ack_packets Int64,
src2dst_psh_packets Int64,
src2dst_rst_packets Int64,
src2dst_fin_packets Int64,
dst2src_syn_packets Int64,
dst2src_cwr_packets Int64,
dst2src_ece_packets Int64,
dst2src_urg_packets Int64,
dst2src_ack_packets Int64,
dst2src_psh_packets Int64,
dst2src_rst_packets Int64,
dst2src_fin_packets Int64,
splt_direction Array(Int64),
splt_ps Array(Int64),
splt_piat_ms Array(Int64),
stf_payload_sizes Array(Int64),
stf_directions Array(Int64),
stf_gram_sequences Array(String),
stf_gram_match_results String,
stf_encrypted_tunnel_flag Int64,
stf_simple_obfs_resp_flag Int64,
stf_valid_packet_counts Int64,
stf_has_tcp_handshake Int64
)
ENGINE =Distributed(ck_cluster,cyber_narrator_galaxy,session_record_nfsp_local,rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_record_nfsp on cluster ck_query (
id Int32,
expiration_id Int32,
src_ip String,
src_mac String,
src_oui String,
src_port Int32,
dst_ip String,
dst_mac String,
dst_oui String,
dst_port Int32,
protocol Int32,
ip_version Int32,
vlan_id Int32,
bidirectional_first_seen_ms Int64,
bidirectional_last_seen_ms Int64,
bidirectional_duration_ms Int64,
bidirectional_packets Int64,
bidirectional_bytes Int64,
src2dst_first_seen_ms Int64,
src2dst_last_seen_ms Int64,
src2dst_duration_ms Int64,
src2dst_packets Int64,
src2dst_bytes Int64,
dst2src_first_seen_ms Int64,
dst2src_last_seen_ms Int64,
dst2src_duration_ms Int64,
dst2src_packets Int64,
dst2src_bytes Int64,
tunnel_id Int64,
application_name String,
application_category_name String,
application_is_guessed Int64,
application_confidence Int64,
requested_server_name String,
client_fingerprint String,
server_fingerprint String,
user_agent String,
content_type String,
bidirectional_min_ps Int64,
bidirectional_mean_ps Float64,
bidirectional_stddev_ps Float64,
bidirectional_max_ps Int64,
src2dst_min_ps Int64,
src2dst_mean_ps Float64,
src2dst_stddev_ps Float64,
src2dst_max_ps Int64,
dst2src_min_ps Int64,
dst2src_mean_ps Float64,
dst2src_stddev_ps Float64,
dst2src_max_ps Int64,
bidirectional_min_piat_ms Int64,
bidirectional_mean_piat_ms Float64,
bidirectional_stddev_piat_ms Float64,
bidirectional_max_piat_ms Int64,
src2dst_min_piat_ms Int64,
src2dst_mean_piat_ms Float64,
src2dst_stddev_piat_ms Float64,
src2dst_max_piat_ms Int64,
dst2src_min_piat_ms Int64,
dst2src_mean_piat_ms Float64,
dst2src_stddev_piat_ms Float64,
dst2src_max_piat_ms Int64,
bidirectional_syn_packets Int64,
bidirectional_cwr_packets Int64,
bidirectional_ece_packets Int64,
bidirectional_urg_packets Int64,
bidirectional_ack_packets Int64,
bidirectional_psh_packets Int64,
bidirectional_rst_packets Int64,
bidirectional_fin_packets Int64,
src2dst_syn_packets Int64,
src2dst_cwr_packets Int64,
src2dst_ece_packets Int64,
src2dst_urg_packets Int64,
src2dst_ack_packets Int64,
src2dst_psh_packets Int64,
src2dst_rst_packets Int64,
src2dst_fin_packets Int64,
dst2src_syn_packets Int64,
dst2src_cwr_packets Int64,
dst2src_ece_packets Int64,
dst2src_urg_packets Int64,
dst2src_ack_packets Int64,
dst2src_psh_packets Int64,
dst2src_rst_packets Int64,
dst2src_fin_packets Int64,
splt_direction Array(Int64),
splt_ps Array(Int64),
splt_piat_ms Array(Int64),
stf_payload_sizes Array(Int64),
stf_directions Array(Int64),
stf_gram_sequences Array(String),
stf_gram_match_results String,
stf_encrypted_tunnel_flag Int64,
stf_simple_obfs_resp_flag Int64,
stf_valid_packet_counts Int64,
stf_has_tcp_handshake Int64
)
ENGINE =Distributed(ck_cluster,cyber_narrator_galaxy,session_record_nfsp_local,rand());

8
tsg_olap/installation/clickhouse/最新全量建表语句/system.sql Normal file
View File

@@ -0,0 +1,8 @@
create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_cluster as `system`.tables ENGINE =Distributed(ck_cluster,`system`,tables,rand());
create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_cluster as `system`.disks ENGINE =Distributed(ck_cluster,`system`,disks,rand());
create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_cluster as `system`.parts ENGINE =Distributed(ck_cluster,`system`,parts,rand());
create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_cluster as `system`.query_log ENGINE =Distributed(ck_cluster,`system`,query_log,rand());
CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_cluster AS `system`.columns ENGINE=Distributed(ck_cluster,`system`,columns,rand());
CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_cluster AS `system`.processes ENGINE=Distributed(ck_cluster,`system`,processes,rand());
alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY;

3231
tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl.sql Normal file
View File

File diff suppressed because it is too large Load Diff

0
tsg_olap/installation/clickhouse/tsg_olap_clickhouse_ddl_check.sql → tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl_check.sql
View File

0
tsg_olap/installation/clickhouse/tsg_olap_clickhouse_ddl_deprecated.sql → tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl_deprecated.sql
View File

0
tsg_olap/installation/clickhouse/system.sql → tsg_olap/installation/clickhouse/最新全量建表语句/旧部署模式建表语句/system.sql
View File

0
tsg_olap/installation/clickhouse/tsg_olap_clickhouse_ddl.sql → tsg_olap/installation/clickhouse/最新全量建表语句/旧部署模式建表语句/tsg_olap_clickhouse_ddl.sql
View File

21
tsg_olap/installation/clickhouse/最新全量建表语句/旧部署模式建表语句/tsg_olap_clickhouse_ddl_check.sql Normal file
View File

@@ -0,0 +1,21 @@
SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT log_id, recv_time, vsys_id, timestamp_us, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

2117
tsg_olap/installation/clickhouse/最新全量建表语句/旧部署模式建表语句/tsg_olap_clickhouse_ddl_deprecated.sql Normal file
View File

File diff suppressed because it is too large Load Diff