From cb3fca18388545053f8c65e82e8bbf3c0b133118 Mon Sep 17 00:00:00 2001
From: houjinchuan <houjinchuan@iie.ac.cn>
Date: Tue, 5 Mar 2024 10:44:44 +0800
Subject: [PATCH] =?UTF-8?q?[CN-1568]=20clickhouse=E5=A2=9E=E5=8A=A0Locatio?=
 =?UTF-8?q?n=20Intelligence=E7=9B=B8=E5=85=B3=E5=BA=93=E8=A1=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../2024/CN-24.03/clickhouse/update-24.03-ck.sql | 48 +++++++++++++++++
 .../start_gohangout_k2ck_location_subscriber_cn.sh | 51 +++++++++++++++++++
 .../gohangout/conf/k2ck_location_subscriber_cn.yml | 28 ++++++++++
 .../2024/CN-24.03/gohangout/start_all.sh      | 23 +++++++++
 .../Clickhouse_CN_建表语句.sql                | 49 +++++++++++++++---
 .../Clickhouse_CN_校验sql.sql                 |  4 +-
 6 files changed, 196 insertions(+), 7 deletions(-)
 create mode 100644 CN发布版本更新记录/2024/CN-24.03/clickhouse/update-24.03-ck.sql
 create mode 100644 CN发布版本更新记录/2024/CN-24.03/gohangout/bin/ghoStart/start_gohangout_k2ck_location_subscriber_cn.sh
 create mode 100644 CN发布版本更新记录/2024/CN-24.03/gohangout/conf/k2ck_location_subscriber_cn.yml
 create mode 100644 CN发布版本更新记录/2024/CN-24.03/gohangout/start_all.sh

diff --git a/CN发布版本更新记录/2024/CN-24.03/clickhouse/update-24.03-ck.sql b/CN发布版本更新记录/2024/CN-24.03/clickhouse/update-24.03-ck.sql
new file mode 100644
index 0000000..c4c97a0
--- /dev/null
+++ b/CN发布版本更新记录/2024/CN-24.03/clickhouse/update-24.03-ck.sql
@@ -0,0 +1,48 @@
+set distributed_ddl_task_timeout = 180;
+
+ALTER table cyber_narrator_galaxy.metric_subscriber_app_local on cluster ck_cluster  drop column IF EXISTS subscriber_longitude, drop column IF EXISTS subscriber_latitude;
+ALTER table cyber_narrator_galaxy.metric_subscriber_app on cluster ck_cluster drop column IF EXISTS subscriber_longitude, drop column IF EXISTS subscriber_latitude;
+ALTER table cyber_narrator_galaxy.metric_subscriber_app on cluster ck_query drop column IF EXISTS subscriber_longitude, drop column IF EXISTS subscriber_latitude;
+
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster (
+ subscriber_id String,
+ imei String,
+ imsi String,
+ phone_number String,
+ apn String,
+ subscriber_longitude Nullable(Float64),
+ subscriber_latitude Nullable(Float64),
+ first_location String,
+ second_location String,
+ third_location String,
+ stat_time Int64
+) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192;
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster (
+ subscriber_id String,
+ imei String,
+ imsi String,
+ phone_number String,
+ apn String,
+ subscriber_longitude Nullable(Float64),
+ subscriber_latitude Nullable(Float64),
+ first_location String,
+ second_location String,
+ third_location String,
+ stat_time Int64
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand());
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_query (
+ subscriber_id String,
+ imei String,
+ imsi String,
+ phone_number String,
+ apn String,
+ subscriber_longitude Nullable(Float64),
+ subscriber_latitude Nullable(Float64),
+ first_location String,
+ second_location String,
+ third_location String,
+ stat_time Int64
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand());
diff --git a/CN发布版本更新记录/2024/CN-24.03/gohangout/bin/ghoStart/start_gohangout_k2ck_location_subscriber_cn.sh b/CN发布版本更新记录/2024/CN-24.03/gohangout/bin/ghoStart/start_gohangout_k2ck_location_subscriber_cn.sh
new file mode 100644
index 0000000..ef63868
--- /dev/null
+++ b/CN发布版本更新记录/2024/CN-24.03/gohangout/bin/ghoStart/start_gohangout_k2ck_location_subscriber_cn.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+#gohangout启动文件
+
+#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下
+YML_NAME=k2ck_location_subscriber_cn
+#gohangout的二进制启动文件路径
+BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
+#日志级别,1,5,10,数字越大日志越详细
+LOG_LV=5
+#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程
+THREAD_SUM=3
+#进程总数
+PROCESS_SUM=$1
+
+if [ ! -d "$BASE_DIR/logs" ]; then
+        mkdir -p $BASE_DIR/logs
+fi
+
+echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+id=0
+logid=0
+while true ; do
+    NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l`
+    pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
+    time_stamp=$(date +%Y%m%d%H%M%S)
+    if [ "${NUM}" -lt ${PROCESS_SUM} ];then
+	$BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 &
+    echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+    ((logid++))
+    ((id++))
+     if [ ${logid} -gt ${PROCESS_SUM} ];then
+      logid=0
+      pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
+        for pid in $pids
+        do
+         kill -9  $pid
+        done
+        echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+        sleep 30
+     fi
+    #大于设置进程数,杀掉所有进程,重启
+    elif [ "${NUM}" -gt ${PROCESS_SUM} ];then
+        for pid in $pids
+        do
+         kill -9  $pid
+        done
+    id=0
+    fi
+        sleep 1
+done
diff --git a/CN发布版本更新记录/2024/CN-24.03/gohangout/conf/k2ck_location_subscriber_cn.yml b/CN发布版本更新记录/2024/CN-24.03/gohangout/conf/k2ck_location_subscriber_cn.yml
new file mode 100644
index 0000000..0e057ee
--- /dev/null
+++ b/CN发布版本更新记录/2024/CN-24.03/gohangout/conf/k2ck_location_subscriber_cn.yml
@@ -0,0 +1,28 @@
+inputs:
+    - Kafka:
+        topic:
+            LOCATION-SUBSCRIBER: 1
+        #assign:
+        #  weblog: [0,9]
+        codec: json
+        consumer_settings:
+            bootstrap.servers: "192.168.44.55:9092"
+            group.id: location_subscriber_1
+            max.partition.fetch.bytes: '10485760'
+            auto.commit.interval.ms: '5000'
+            # from.beginning: 'true'
+            # sasl.mechanism: PLAIN
+            # sasl.user: admin
+            # sasl.password: admin-secret
+
+outputs:
+    - Clickhouse:
+        table: 'cyber_narrator_galaxy.location_subscriber_local'
+        username: 'default'
+        password: 'ceiec2019'
+        hosts:
+        - 'tcp://192.168.44.55:9001'
+        bulk_actions: 100000
+        flush_interval: 30
+        concurrent: 2
+        conn_max_life_time: 60
diff --git a/CN发布版本更新记录/2024/CN-24.03/gohangout/start_all.sh b/CN发布版本更新记录/2024/CN-24.03/gohangout/start_all.sh
new file mode 100644
index 0000000..cca4927
--- /dev/null
+++ b/CN发布版本更新记录/2024/CN-24.03/gohangout/start_all.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+STARTDIR=$(cd $(dirname $0); pwd)
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_location_subscriber_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_subscriber_app_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_security_event_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_region_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_asn_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_application_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_domain_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_protocol_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_link_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qname_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_server_ip_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qtype_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rcode_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_a_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_aaaa_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_cname_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh $1 > /dev/null 2>&1 &
+$STARTDIR/bin/ghoStart/start_gohangout_k2ck_cn_record.sh $1
diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql
index ee2acff..5b7d244 100644
--- a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql
+++ b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql
@@ -1161,8 +1161,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app_local ON
  imsi String,
  phone_number String,
  apn String,
- subscriber_longitude Nullable(Float64),
- subscriber_latitude Nullable(Float64),
  stat_time Int64,
  sent_pkts Int64,
  sent_bytes Int64,
@@ -1201,8 +1199,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE
  imsi String,
  phone_number String,
  apn String,
- subscriber_longitude Nullable(Float64),
- subscriber_latitude Nullable(Float64),
  stat_time Int64,
  sent_pkts Int64,
  sent_bytes Int64,
@@ -1241,8 +1237,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE
  imsi String,
  phone_number String,
  apn String,
- subscriber_longitude Nullable(Float64),
- subscriber_latitude Nullable(Float64),
  stat_time Int64,
  sent_pkts Int64,
  sent_bytes Int64,
@@ -1275,6 +1269,49 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE
 ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand());
 
 
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster (
+ subscriber_id String,
+ imei String,
+ imsi String,
+ phone_number String,
+ apn String,
+ subscriber_longitude Nullable(Float64),
+ subscriber_latitude Nullable(Float64),
+ first_location String,
+ second_location String,
+ third_location String,
+ stat_time Int64
+) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192;
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster (
+ subscriber_id String,
+ imei String,
+ imsi String,
+ phone_number String,
+ apn String,
+ subscriber_longitude Nullable(Float64),
+ subscriber_latitude Nullable(Float64),
+ first_location String,
+ second_location String,
+ third_location String,
+ stat_time Int64
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand());
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_query (
+ subscriber_id String,
+ imei String,
+ imsi String,
+ phone_number String,
+ apn String,
+ subscriber_longitude Nullable(Float64),
+ subscriber_latitude Nullable(Float64),
+ first_location String,
+ second_location String,
+ third_location String,
+ stat_time Int64
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand());
+
+
 CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_server_ip_local ON CLUSTER ck_cluster (
  server_ip String,
  server_country_region String,
diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql
index a62d82b..082b2e6 100644
--- a/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql
+++ b/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql
@@ -43,8 +43,10 @@ SELECT l7_protocol, server_port, stat_time, sent_pkts, sent_bytes, received_pkts
 FROM cyber_narrator_galaxy.metric_protocol where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT country_region, super_admin_area, admin_area, side, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
 FROM cyber_narrator_galaxy.metric_region where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT subscriber_id, app, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
+SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
 FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,  stat_time
+FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
 FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');