From c0a9a6099803993752ea9ca8f756753f02954126 Mon Sep 17 00:00:00 2001
From: houjinchuan <houjinchuan@iie.ac.cn>
Date: Mon, 8 Apr 2024 10:36:24 +0800
Subject: [PATCH] =?UTF-8?q?[CN-1620]=20Tag=E9=A2=84=E8=81=9A=E5=90=88?=
 =?UTF-8?q?=E5=BA=93=E8=A1=A8=E5=88=9B=E5=BB=BA?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../2024/CN-24.04/clickhouse/check-24.04-ck.sql |  4 ++-
 .../2024/CN-24.04/clickhouse/update-24.04-ck.sql | 32 +++++++++++++++++++
 .../Clickhouse_CN_建表语句.sql                | 32 +++++++++++++++++++
 .../Clickhouse_CN_校验sql.sql                 |  2 ++
 4 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/CN发布版本更新记录/2024/CN-24.04/clickhouse/check-24.04-ck.sql b/CN发布版本更新记录/2024/CN-24.04/clickhouse/check-24.04-ck.sql
index 0642d38..4697cab 100644
--- a/CN发布版本更新记录/2024/CN-24.04/clickhouse/check-24.04-ck.sql
+++ b/CN发布版本更新记录/2024/CN-24.04/clickhouse/check-24.04-ck.sql
@@ -15,4 +15,6 @@ FROM cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric where crea
 SELECT app_name,subscriber_id,imei,imsi,phone_number,apn,app_category,app_subcategory,create_time,update_time
 FROM cyber_narrator_galaxy.session_relation_subscriber_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT ip, l7_protocol, port, create_time, update_time
-FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
\ No newline at end of file
+FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state 
+FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
diff --git a/CN发布版本更新记录/2024/CN-24.04/clickhouse/update-24.04-ck.sql b/CN发布版本更新记录/2024/CN-24.04/clickhouse/update-24.04-ck.sql
index f8172b3..832edab 100644
--- a/CN发布版本更新记录/2024/CN-24.04/clickhouse/update-24.04-ck.sql
+++ b/CN发布版本更新记录/2024/CN-24.04/clickhouse/update-24.04-ck.sql
@@ -297,3 +297,35 @@ GROUP BY
     ip,
     l7_protocol,
     port;
+
+
+CREATE TABLE cyber_narrator_galaxy.metric_tag_local
+(
+    tag String,
+    stat_time Int64,
+    ip_sketch String,
+    domain_sketch String,
+    ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
+    domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(stat_time))
+ORDER BY (stat_time, tag);
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_cluster (
+    tag String,
+    stat_time Int64,
+    ip_sketch String,
+    domain_sketch String,
+    ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
+    domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_query (
+    tag String,
+    stat_time Int64,
+    ip_sketch String,
+    domain_sketch String,
+    ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
+    domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql
index 170b391..38e62e2 100644
--- a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql
+++ b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql
@@ -1269,6 +1269,38 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE
 ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand());
 
 
+CREATE TABLE cyber_narrator_galaxy.metric_tag_local
+(
+    tag String,
+    stat_time Int64,
+    ip_sketch String,
+    domain_sketch String,
+    ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
+    domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(stat_time))
+ORDER BY (stat_time, tag);
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_cluster (
+    tag String,
+    stat_time Int64,
+    ip_sketch String,
+    domain_sketch String,
+    ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
+    domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_query (
+    tag String,
+    stat_time Int64,
+    ip_sketch String,
+    domain_sketch String,
+    ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
+    domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
+
+
 CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster (
  subscriber_id String,
  imei String,
diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql
index fbfaa53..cb72177 100644
--- a/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql
+++ b/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql
@@ -49,6 +49,8 @@ SELECT country_region, super_admin_area, admin_area, side, stat_time, sent_pkts,
 FROM cyber_narrator_galaxy.metric_region where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
 FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state 
+FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
 FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms