From 5807554e63d0a598bc24f255d49a9ec24a1259fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E4=BE=AF=E6=99=8B=E5=B7=9D?=
 <houjinchuan@geedgenetworks.com>
Date: Fri, 1 Nov 2024 18:40:47 +0800
Subject: [PATCH] =?UTF-8?q?[CN-1714]=20session=5Frelation=5Fsubscriber=5Fa?=
 =?UTF-8?q?pp=E5=BA=93=E8=A1=A8=E6=96=B0=E5=A2=9E=E5=AD=97=E6=AE=B5entity?=
 =?UTF-8?q?=5Ftags?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../clickhouse/cn_clickhouse_ddl.sql          |  9 +++-
 .../clickhouse/cn_clickhouse_ddl_check.sql    |  4 +-
 .../clickhouse/cn_clickhouse_ddl_24.08.sql    |  9 +++-
 .../cn_clickhouse_ddl_check_24.08.sql         |  4 +-
 .../cn_clickhouse_ddl_upgrade_24.08.sql       | 54 +++++++++++++++++++
 5 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql b/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql
index 056f6ac..26ffd1a 100644
--- a/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql
+++ b/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql
@@ -1210,6 +1210,7 @@ ENGINE = Distributed('ck_cluster',
  apn String,
  app_category String,
  app_subcategory String,
+ entity_tags Array(String),
  create_time Int64,
  update_time Int64
 ) ENGINE = MergeTree
@@ -1224,7 +1225,8 @@ ENGINE = Distributed('ck_cluster',
  phone_number = anyLast(phone_number),
  apn = anyLast(apn),
  app_category = anyLast(app_category),
- app_subcategory = anyLast(app_subcategory);
+ app_subcategory = anyLast(app_subcategory),
+ entity_tags = groupUniqArrayArray(entity_tags);
 
 CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_cluster (
  app_name String,
@@ -1235,6 +1237,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app
  apn String,
  app_category String,
  app_subcategory String,
+ entity_tags Array(String),
  create_time Int64,
  update_time Int64
 ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
@@ -1249,6 +1252,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber
  apn String,
  app_category String,
  app_subcategory String,
+ entity_tags Array(String),
  stat_time Int64
 ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192;
 
@@ -1261,6 +1265,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber
  apn String,
  app_category String,
  app_subcategory String,
+ entity_tags Array(String),
  stat_time Int64
 ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
 
@@ -1451,6 +1456,7 @@ TO cyber_narrator_galaxy.session_relation_subscriber_app_local
     apn String,
     app_category String,
     app_subcategory String,
+    entity_tags Array(String),
     create_time Int64,
     update_time Int64
 ) AS
@@ -1463,6 +1469,7 @@ SELECT
     anyLast(apn) AS apn,
     anyLast(app_category) AS app_category,
     anyLast(app_subcategory) AS app_subcategory,
+    groupUniqArrayArray(entity_tags) AS entity_tags,
     min(c1.stat_time) AS create_time,
     max(c1.stat_time) AS update_time
 FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app_local AS c1
diff --git a/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl_check.sql b/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl_check.sql
index 07797da..f799fb1 100644
--- a/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl_check.sql
+++ b/cyber_narrator/installation/clickhouse/cn_clickhouse_ddl_check.sql
@@ -6,9 +6,9 @@ SELECT status, is_builtin, rule_type, victim_ip, offender_ip, event_info, event_
 FROM cyber_narrator_galaxy.cn_security_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT app_name ,domain ,ip ,ip_country_region ,ip_super_admin_area ,ip_admin_area ,ip_asn ,ip_isp ,domain_category_name ,domain_category_group ,app_category ,app_subcategory ,entity_tags ,stat_time 
 FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT  app_name,  subscriber_id,  imei,  imsi,  phone_number,  apn,  app_category,  app_subcategory,  create_time,  update_time
+SELECT  app_name,  subscriber_id,  imei,  imsi,  phone_number,  apn,  app_category,  app_subcategory,  create_time,  update_time , entity_tags
 FROM cyber_narrator_galaxy.session_relation_subscriber_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT  app_name ,  subscriber_id ,  imei ,  imsi ,  phone_number ,  apn ,  app_category ,  app_subcategory ,  stat_time 
+SELECT  app_name ,  subscriber_id ,  imei ,  imsi ,  phone_number ,  apn ,  app_category ,  app_subcategory ,  stat_time , entity_tags
 FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
 FROM cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
diff --git a/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_24.08.sql b/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_24.08.sql
index 056f6ac..26ffd1a 100644
--- a/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_24.08.sql
+++ b/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_24.08.sql
@@ -1210,6 +1210,7 @@ ENGINE = Distributed('ck_cluster',
  apn String,
  app_category String,
  app_subcategory String,
+ entity_tags Array(String),
  create_time Int64,
  update_time Int64
 ) ENGINE = MergeTree
@@ -1224,7 +1225,8 @@ ENGINE = Distributed('ck_cluster',
  phone_number = anyLast(phone_number),
  apn = anyLast(apn),
  app_category = anyLast(app_category),
- app_subcategory = anyLast(app_subcategory);
+ app_subcategory = anyLast(app_subcategory),
+ entity_tags = groupUniqArrayArray(entity_tags);
 
 CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_cluster (
  app_name String,
@@ -1235,6 +1237,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app
  apn String,
  app_category String,
  app_subcategory String,
+ entity_tags Array(String),
  create_time Int64,
  update_time Int64
 ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
@@ -1249,6 +1252,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber
  apn String,
  app_category String,
  app_subcategory String,
+ entity_tags Array(String),
  stat_time Int64
 ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192;
 
@@ -1261,6 +1265,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber
  apn String,
  app_category String,
  app_subcategory String,
+ entity_tags Array(String),
  stat_time Int64
 ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
 
@@ -1451,6 +1456,7 @@ TO cyber_narrator_galaxy.session_relation_subscriber_app_local
     apn String,
     app_category String,
     app_subcategory String,
+    entity_tags Array(String),
     create_time Int64,
     update_time Int64
 ) AS
@@ -1463,6 +1469,7 @@ SELECT
     anyLast(apn) AS apn,
     anyLast(app_category) AS app_category,
     anyLast(app_subcategory) AS app_subcategory,
+    groupUniqArrayArray(entity_tags) AS entity_tags,
     min(c1.stat_time) AS create_time,
     max(c1.stat_time) AS update_time
 FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app_local AS c1
diff --git a/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_check_24.08.sql b/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_check_24.08.sql
index 07797da..f799fb1 100644
--- a/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_check_24.08.sql
+++ b/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_check_24.08.sql
@@ -6,9 +6,9 @@ SELECT status, is_builtin, rule_type, victim_ip, offender_ip, event_info, event_
 FROM cyber_narrator_galaxy.cn_security_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT app_name ,domain ,ip ,ip_country_region ,ip_super_admin_area ,ip_admin_area ,ip_asn ,ip_isp ,domain_category_name ,domain_category_group ,app_category ,app_subcategory ,entity_tags ,stat_time 
 FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT  app_name,  subscriber_id,  imei,  imsi,  phone_number,  apn,  app_category,  app_subcategory,  create_time,  update_time
+SELECT  app_name,  subscriber_id,  imei,  imsi,  phone_number,  apn,  app_category,  app_subcategory,  create_time,  update_time , entity_tags
 FROM cyber_narrator_galaxy.session_relation_subscriber_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT  app_name ,  subscriber_id ,  imei ,  imsi ,  phone_number ,  apn ,  app_category ,  app_subcategory ,  stat_time 
+SELECT  app_name ,  subscriber_id ,  imei ,  imsi ,  phone_number ,  apn ,  app_category ,  app_subcategory ,  stat_time , entity_tags
 FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
 FROM cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
diff --git a/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_upgrade_24.08.sql b/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_upgrade_24.08.sql
index b28dc2d..c64e14c 100644
--- a/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_upgrade_24.08.sql
+++ b/cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_upgrade_24.08.sql
@@ -224,3 +224,57 @@ ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add
 
 ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;
 ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;
+
+
+ALTER table cyber_narrator_galaxy.session_relation_subscriber_app_local ON CLUSTER ck_cluster add column IF NOT EXISTS entity_tags Array(String) after app_subcategory;
+ALTER table cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_cluster add column IF NOT EXISTS entity_tags Array(String) after app_subcategory;
+
+ALTER table cyber_narrator_galaxy.raw_session_relation_subscriber_app_local ON CLUSTER ck_cluster add column IF NOT EXISTS entity_tags Array(String) after app_subcategory;
+ALTER table cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_cluster add column IF NOT EXISTS entity_tags Array(String) after app_subcategory;
+
+ALTER table cyber_narrator_galaxy.session_relation_subscriber_app_local on cluster ck_cluster MODIFY 
+ TTL toDateTime(update_time) + toIntervalSecond(2592000),
+ toDateTime(update_time) + toIntervalSecond(1) GROUP BY subscriber_id,
+ app_name SET create_time = min(create_time),
+ update_time = max(update_time),
+ imei = anyLast(imei),
+ imsi = anyLast(imsi),
+ phone_number = anyLast(phone_number),
+ apn = anyLast(apn),
+ app_category = anyLast(app_category),
+ app_subcategory = anyLast(app_subcategory),
+entity_tags = groupUniqArrayArray(entity_tags);
+
+DROP VIEW IF EXISTS cyber_narrator_galaxy.session_relation_subscriber_app_view_metric on cluster ck_cluster;
+
+CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_subscriber_app_view_metric on cluster ck_cluster 
+TO cyber_narrator_galaxy.session_relation_subscriber_app_local
+(
+    app_name String,
+    subscriber_id String,
+    imei String,
+    imsi String,
+    phone_number String,
+    apn String,
+    app_category String,
+    app_subcategory String,
+    entity_tags Array(String),
+    create_time Int64,
+    update_time Int64
+) AS
+SELECT
+    app_name AS app_name,
+    subscriber_id AS subscriber_id,
+    anyLast(imei) AS imei,
+    anyLast(imsi) AS imsi,
+    anyLast(phone_number) AS phone_number,
+    anyLast(apn) AS apn,
+    anyLast(app_category) AS app_category,
+    anyLast(app_subcategory) AS app_subcategory,
+    groupUniqArrayArray(entity_tags) AS entity_tags,
+    min(c1.stat_time) AS create_time,
+    max(c1.stat_time) AS update_time
+FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app_local AS c1
+GROUP BY
+    subscriber_id,
+    app_name;   
\ No newline at end of file