diff --git a/CN发布版本更新记录/2024/CN-24.03/clickhouse/check-24.03-ck.sql b/CN发布版本更新记录/2024/CN-24.03/clickhouse/check-24.03-ck.sql new file mode 100644 index 0000000..ade9af6 --- /dev/null +++ b/CN发布版本更新记录/2024/CN-24.03/clickhouse/check-24.03-ck.sql @@ -0,0 +1,6 @@ +set distributed_ddl_task_timeout = 180; + +SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms +FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time > $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/CN发布版本更新记录/2024/CN-24.04/gohangout/bin/start_gohangout_k2ck_raw_session_relation_domain_ip_app_cn.sh b/CN发布版本更新记录/2024/CN-24.04/gohangout/bin/start_gohangout_k2ck_raw_session_relation_domain_ip_app_cn.sh new file mode 100644 index 0000000..34ed3a9 --- /dev/null +++ b/CN发布版本更新记录/2024/CN-24.04/gohangout/bin/start_gohangout_k2ck_raw_session_relation_domain_ip_app_cn.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_raw_session_relation_domain_ip_app_cn +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=5 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/CN发布版本更新记录/2024/CN-24.04/gohangout/bin/start_gohangout_k2ck_raw_session_relation_subscriber_app_cn.sh b/CN发布版本更新记录/2024/CN-24.04/gohangout/bin/start_gohangout_k2ck_raw_session_relation_subscriber_app_cn.sh new file mode 100644 index 0000000..63cadcd --- /dev/null +++ b/CN发布版本更新记录/2024/CN-24.04/gohangout/bin/start_gohangout_k2ck_raw_session_relation_subscriber_app_cn.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_raw_session_relation_subscriber_app_cn +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=5 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_cn_ip_dynamic_attribute_cn.yml b/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_cn_ip_dynamic_attribute_cn.yml new file mode 100644 index 0000000..58acb5f --- /dev/null +++ b/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_cn_ip_dynamic_attribute_cn.yml @@ -0,0 +1,28 @@ +inputs: + - Kafka: + topic: + RAW-CN-IP-DYNAMIC-ATTRIBUTE: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "192.168.44.55:9092" + group.id: tsg-raw_cn_ip_dynamic_attribute_cn + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local' + username: 'default' + password: 'ceiec2019' + hosts: + - 'tcp://192.168.44.55:9001' + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_session_relation_domain_ip_app_cn.yml b/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_session_relation_domain_ip_app_cn.yml new file mode 100644 index 0000000..fcb9c50 --- /dev/null +++ b/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_session_relation_domain_ip_app_cn.yml @@ -0,0 +1,28 @@ +inputs: + - Kafka: + topic: + RAW-SESSION-RELATION-DOMAIN-IP-APP: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "192.168.44.55:9092" + group.id: tsg-raw_session_relation_domain_ip_app_cn + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local' + username: 'default' + password: 'ceiec2019' + hosts: + - 'tcp://192.168.44.55:9001' + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_session_relation_subscriber_app_cn.yml b/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_session_relation_subscriber_app_cn.yml new file mode 100644 index 0000000..41f4730 --- /dev/null +++ b/CN发布版本更新记录/2024/CN-24.04/gohangout/conf/k2ck_raw_session_relation_subscriber_app_cn.yml @@ -0,0 +1,28 @@ +inputs: + - Kafka: + topic: + RAW-SESSION-RELATION-SUBSCRIBER-APP: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "192.168.44.55:9092" + group.id: tsg-raw_session_relation_subscriber_app_cn + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'cyber_narrator_galaxy.raw_session_relation_subscriber_app_local' + username: 'default' + password: 'ceiec2019' + hosts: + - 'tcp://192.168.44.55:9001' + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/CN发布版本更新记录/2024/CN-24.04/start_all.sh b/CN发布版本更新记录/2024/CN-24.04/start_all.sh new file mode 100644 index 0000000..95b34f1 --- /dev/null +++ b/CN发布版本更新记录/2024/CN-24.04/start_all.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +STARTDIR=$(cd $(dirname $0); pwd) +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_location_subscriber_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_subscriber_app_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_security_event_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_region_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_asn_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_application_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_domain_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_protocol_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_link_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qname_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_server_ip_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qtype_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rcode_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_a_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_aaaa_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_cname_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_raw_session_relation_domain_ip_app_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_raw_session_relation_subscriber_app_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_raw_cn_ip_dynamic_attribute_cn.sh $1 > /dev/null 2>&1 & +$STARTDIR/bin/ghoStart/start_gohangout_k2ck_cn_record.sh $1 diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql index 8a01062..170b391 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql @@ -1280,6 +1280,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL first_location String, second_location String, third_location String, + data_source String, stat_time Int64 ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192; @@ -1294,6 +1295,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER first_location String, second_location String, third_location String, + data_source String, stat_time Int64 ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand()); @@ -1308,6 +1310,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER first_location String, second_location String, third_location String, + data_source String, stat_time Int64 ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'location_subscriber_local', rand()); @@ -1571,7 +1574,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_dns_rr_cname ON CLUSTER ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_dns_rr_cname_local', rand()); -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation_local ON CLUSTER ck_cluster ( +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local ON CLUSTER ck_cluster ( app_name String, domain String, ip String, @@ -1588,7 +1591,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation_local ON CLUSTE stat_time Int64 ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192; -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation ON CLUSTER ck_cluster ( +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_cluster ( app_name String, domain String, ip String, @@ -1603,9 +1606,9 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation ON CLUSTER ck_c app_subcategory String, entity_tags Array(String), stat_time Int64 -) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_relation_local', rand()); +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand()); -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation ON CLUSTER ck_query ( +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_query ( app_name String, domain String, ip String, @@ -1620,34 +1623,33 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_relation ON CLUSTER ck_q app_subcategory String, entity_tags Array(String), stat_time Int64 -) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_relation_local', rand()); +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand()); -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute_local ON CLUSTER ck_cluster ( +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local ON CLUSTER ck_cluster ( ip String, l7_protocol String, port Int64, stat_time Int64 ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192; -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute ON CLUSTER ck_cluster ( +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_cluster ( ip String, l7_protocol String, port Int64, stat_time Int64 -) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_ip_dynamic_attribute_local', rand()); +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand()); -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute ON CLUSTER ck_query ( +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_query ( ip String, l7_protocol String, port Int64, stat_time Int64 -) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_ip_dynamic_attribute_local', rand()); +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand()); -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation_local on cluster ck_cluster +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app_local on cluster ck_cluster ( - app_name String, domain String, ip String, @@ -1684,9 +1686,8 @@ TTL toDateTime(update_time) + toIntervalSecond(2592000), app_subcategory = anyLast(app_subcategory), entity_tags = groupUniqArrayArray(entity_tags); -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_query +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_query ( - app_name String, domain String, ip String, @@ -1705,12 +1706,11 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster c ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', - 'cn_entity_relation_local', + 'session_relation_domain_ip_app_local', rand()); - CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_cluster + CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_cluster ( - app_name String, domain String, ip String, @@ -1729,93 +1729,98 @@ ENGINE = Distributed('ck_cluster', ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', - 'cn_entity_relation_local', + 'session_relation_domain_ip_app_local', rand()); - - -CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_entity_relation_view TO cyber_narrator_galaxy.cn_entity_relation_local -( - app_name String, - domain String, - ip String, - ip_country_region String, - ip_super_admin_area String, - ip_admin_area String, - ip_asn String, - ip_isp String, - domain_category_name String, - domain_category_group String, - app_category String, - app_subcategory String, - entity_tags Array(String), - create_time Int64, - update_time Int64 -) AS -SELECT - app AS app_name, - domain AS domain, - server_ip AS ip, - anyLast(server_country_region) AS ip_country_region, - anyLast(server_super_admin_area) AS ip_super_admin_area, - anyLast(server_admin_area) AS ip_admin_area, - anyLast(server_asn) AS ip_asn, - anyLast(server_isp) AS ip_isp, - anyLast(domain_category_name) AS domain_category_name, - anyLast(domain_category_group) AS domain_category_group, - anyLast(app_category) AS app_category, - anyLast(app_subcategory) AS app_subcategory, - groupUniqArrayArray(arrayConcat(server_ip_tags, domain_tags, app_tags)) AS entity_tags, - min(c1.recv_time) AS create_time, - max(c1.recv_time) AS update_time -FROM cyber_narrator_galaxy.session_record_cn_local AS c1 -WHERE (decoded_path LIKE '%IPv4_TCP%') OR (server_port IN (53, 443)) -GROUP BY - ip, - app_name, - domain -; - - CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_entity_relation_view_metric on cluster ck_cluster TO cyber_narrator_galaxy.cn_entity_relation_local -( - - app_name String, - domain String, - ip String, - ip_country_region String, - ip_super_admin_area String, - ip_admin_area String, - ip_asn String, - ip_isp String, - domain_category_name String, - domain_category_group String, - app_category String, - app_subcategory String, - entity_tags Array(String), - create_time Int64, - update_time Int64 -) AS -SELECT - app_name AS app_name, - domain AS domain, - ip AS ip, - anyLast(ip_country_region) AS ip_country_region, - anyLast(ip_super_admin_area) AS ip_super_admin_area, - anyLast(ip_admin_area) AS ip_admin_area, - anyLast(ip_asn) AS ip_asn, - anyLast(ip_isp) AS ip_isp, - anyLast(domain_category_name) AS domain_category_name, - anyLast(domain_category_group) AS domain_category_group, - anyLast(app_category) AS app_category, - anyLast(app_subcategory) AS app_subcategory, - groupUniqArrayArray(entity_tags) AS entity_tags, - min(c1.stat_time) AS create_time, - max(c1.stat_time) AS update_time -FROM cyber_narrator_galaxy.metric_relation_local AS c1 -GROUP BY - ip, - app_name, - domain; + + CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app_local ON CLUSTER ck_cluster ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + create_time Int64, + update_time Int64 +) ENGINE = MergeTree + ORDER BY (subscriber_id, app_name) + TTL toDateTime(update_time) + toIntervalSecond(2592000), + toDateTime(update_time) + toIntervalSecond(1) + GROUP BY subscriber_id, app_name + SET create_time = min(create_time), + update_time = max(update_time), + imei = anyLast(imei), + imsi = anyLast(imsi), + phone_number = anyLast(phone_number), + apn = anyLast(apn), + app_category = anyLast(app_category), + app_subcategory = anyLast(app_subcategory); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_cluster ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + create_time Int64, + update_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_query ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + create_time Int64, + update_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand()); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app_local ON CLUSTER ck_cluster ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + stat_time Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time) SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_cluster ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_query ( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + stat_time Int64 +) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand()); + CREATE TABLE if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_local on cluster ck_cluster ( @@ -1860,57 +1865,15 @@ ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'cn_ip_dynamic_attribute_local', rand()); - - -CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_view TO cyber_narrator_galaxy.cn_ip_dynamic_attribute_local -( - ip String, - l7_protocol String, - port Int64, - create_time Int64, - update_time Int64 -) AS -SELECT - server_ip AS ip, - l7_protocol AS l7_protocol, - server_port AS port, - min(c1.recv_time) AS create_time, - max(c1.recv_time) AS update_time -FROM cyber_narrator_galaxy.session_record_cn_local AS c1 -WHERE (decoded_path LIKE '%IPv4_TCP%') OR (server_port IN (53, 443)) -GROUP BY - ip, - l7_protocol, - port -; - - CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric on cluster ck_cluster TO cyber_narrator_galaxy.cn_ip_dynamic_attribute_local -( - ip String, - l7_protocol String, - port Int64, - create_time Int64, - update_time Int64 -) AS -SELECT - ip AS ip, - l7_protocol AS l7_protocol, - port AS port, - min(c1.stat_time) AS create_time, - max(c1.stat_time) AS update_time -FROM cyber_narrator_galaxy.metric_ip_dynamic_attribute_local AS c1 -GROUP BY - ip, - l7_protocol, - port; - -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event_local ON CLUSTER ck_cluster + + + CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event_local ON CLUSTER ck_cluster ( status Int64, is_builtin Int64, rule_type String, victim_ip String, - victim_country_region String, + victim_country_region String, victim_super_admin_area String, victim_admin_area String, victim_longitude Nullable(Float64), @@ -1968,7 +1931,6 @@ TTL toDateTime(end_time) + toIntervalSecond(2592000), end_time = max(end_time), match_times = anyLast(match_times); - CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_cluster ( status Int64, @@ -2040,3 +2002,96 @@ ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'cn_security_event_local', rand()); + + + CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric on cluster ck_cluster +TO cyber_narrator_galaxy.session_relation_domain_ip_app_local +( + app_name String, + domain String, + ip String, + ip_country_region String, + ip_super_admin_area String, + ip_admin_area String, + ip_asn String, + ip_isp String, + domain_category_name String, + domain_category_group String, + app_category String, + app_subcategory String, + entity_tags Array(String), + create_time Int64, + update_time Int64 +) AS +SELECT + app_name AS app_name, + domain AS domain, + ip AS ip, + anyLast(ip_country_region) AS ip_country_region, + anyLast(ip_super_admin_area) AS ip_super_admin_area, + anyLast(ip_admin_area) AS ip_admin_area, + anyLast(ip_asn) AS ip_asn, + anyLast(ip_isp) AS ip_isp, + anyLast(domain_category_name) AS domain_category_name, + anyLast(domain_category_group) AS domain_category_group, + anyLast(app_category) AS app_category, + anyLast(app_subcategory) AS app_subcategory, + groupUniqArrayArray(entity_tags) AS entity_tags, + min(c1.stat_time) AS create_time, + max(c1.stat_time) AS update_time +FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app_local AS c1 +GROUP BY + ip, + app_name, + domain; + +CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_subscriber_app_view_metric on cluster ck_cluster +TO cyber_narrator_galaxy.session_relation_subscriber_app_local +( + app_name String, + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + app_category String, + app_subcategory String, + create_time Int64, + update_time Int64 +) AS +SELECT + app_name AS app_name, + subscriber_id AS subscriber_id, + anyLast(imei) AS imei, + anyLast(imsi) AS imsi, + anyLast(phone_number) AS phone_number, + anyLast(apn) AS apn, + anyLast(app_category) AS app_category, + anyLast(app_subcategory) AS app_subcategory, + min(c1.stat_time) AS create_time, + max(c1.stat_time) AS update_time +FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app_local AS c1 +GROUP BY + subscriber_id, + app_name; + +CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric on cluster ck_cluster +TO cyber_narrator_galaxy.cn_ip_dynamic_attribute_local +( + ip String, + l7_protocol String, + port Int64, + create_time Int64, + update_time Int64 +) AS +SELECT + ip AS ip, + l7_protocol AS l7_protocol, + port AS port, + min(c1.stat_time) AS create_time, + max(c1.stat_time) AS update_time +FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute_local AS c1 +GROUP BY + ip, + l7_protocol, + port; diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql index 082b2e6..fbfaa53 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_CN_校验sql.sql @@ -1,17 +1,21 @@ SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time -FROM cyber_narrator_galaxy.cn_entity_relation where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND start_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND create_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('203 SELECT ip, side, `zone`, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms, asymmetric_sessions, bulky_sessions, cbr_streaming_sessions, download_sessions, interactive_sessions, pseudo_unidirectional_sessions, streaming_sessions, unidirectional_sessions, random_looking_sessions, bidirectional_sessions FROM cyber_narrator_galaxy.metric_ip where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND stat_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time