diff --git a/CN发布版本更新记录/CN-23.12/clickhouse/update-23.12-ck.sql b/CN发布版本更新记录/CN-23.12/clickhouse/update-23.12-ck.sql index 6fa7aff..7da3dca 100644 --- a/CN发布版本更新记录/CN-23.12/clickhouse/update-23.12-ck.sql +++ b/CN发布版本更新记录/CN-23.12/clickhouse/update-23.12-ck.sql @@ -538,121 +538,15 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_protocol ON CLUSTER ck_q ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_protocol_local', rand()); -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_local ON CLUSTER ck_cluster ( - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - stat_time Int64, - sent_pkts Int64, - sent_bytes Int64, - received_pkts Int64, - received_bytes Int64, - sessions Int64, - traffic_inbound_byte Int64, - traffic_inbound_pkt Int64, - traffic_outbound_byte Int64, - traffic_outbound_pkt Int64, - traffic_internal_byte Int64, - traffic_internal_pkt Int64, - traffic_through_byte Int64, - traffic_through_pkt Int64, - tcp_c2s_lost_bytes_ratio Nullable(Float64), - tcp_s2c_lost_bytes_ratio Nullable(Float64), - tcp_lost_bytes_ratio Nullable(Float64), - tcp_c2s_o3_pkts_ratio Nullable(Float64), - tcp_s2c_o3_pkts_ratio Nullable(Float64), - tcp_o3_pkts_ratio Nullable(Float64), - tcp_c2s_rtx_bytes_ratio Nullable(Float64), - tcp_s2c_rtx_bytes_ratio Nullable(Float64), - tcp_rtx_bytes_ratio Nullable(Float64), - tcp_c2s_rtx_pkts_ratio Nullable(Float64), - tcp_s2c_rtx_pkts_ratio Nullable(Float64), - tcp_rtx_pkts_ratio Nullable(Float64), - avg_tcp_rtt_ms Nullable(Float64), - avg_http_response_latency_ms Nullable(Float64), - avg_ssl_handshake_latency_ms Nullable(Float64) -) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192; - -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber ON CLUSTER ck_cluster ( - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - stat_time Int64, - sent_pkts Int64, - sent_bytes Int64, - received_pkts Int64, - received_bytes Int64, - sessions Int64, - traffic_inbound_byte Int64, - traffic_inbound_pkt Int64, - traffic_outbound_byte Int64, - traffic_outbound_pkt Int64, - traffic_internal_byte Int64, - traffic_internal_pkt Int64, - traffic_through_byte Int64, - traffic_through_pkt Int64, - tcp_c2s_lost_bytes_ratio Nullable(Float64), - tcp_s2c_lost_bytes_ratio Nullable(Float64), - tcp_lost_bytes_ratio Nullable(Float64), - tcp_c2s_o3_pkts_ratio Nullable(Float64), - tcp_s2c_o3_pkts_ratio Nullable(Float64), - tcp_o3_pkts_ratio Nullable(Float64), - tcp_c2s_rtx_bytes_ratio Nullable(Float64), - tcp_s2c_rtx_bytes_ratio Nullable(Float64), - tcp_rtx_bytes_ratio Nullable(Float64), - tcp_c2s_rtx_pkts_ratio Nullable(Float64), - tcp_s2c_rtx_pkts_ratio Nullable(Float64), - tcp_rtx_pkts_ratio Nullable(Float64), - avg_tcp_rtt_ms Nullable(Float64), - avg_http_response_latency_ms Nullable(Float64), - avg_ssl_handshake_latency_ms Nullable(Float64) -) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_local', rand()); - -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber ON CLUSTER ck_query ( - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - stat_time Int64, - sent_pkts Int64, - sent_bytes Int64, - received_pkts Int64, - received_bytes Int64, - sessions Int64, - traffic_inbound_byte Int64, - traffic_inbound_pkt Int64, - traffic_outbound_byte Int64, - traffic_outbound_pkt Int64, - traffic_internal_byte Int64, - traffic_internal_pkt Int64, - traffic_through_byte Int64, - traffic_through_pkt Int64, - tcp_c2s_lost_bytes_ratio Nullable(Float64), - tcp_s2c_lost_bytes_ratio Nullable(Float64), - tcp_lost_bytes_ratio Nullable(Float64), - tcp_c2s_o3_pkts_ratio Nullable(Float64), - tcp_s2c_o3_pkts_ratio Nullable(Float64), - tcp_o3_pkts_ratio Nullable(Float64), - tcp_c2s_rtx_bytes_ratio Nullable(Float64), - tcp_s2c_rtx_bytes_ratio Nullable(Float64), - tcp_rtx_bytes_ratio Nullable(Float64), - tcp_c2s_rtx_pkts_ratio Nullable(Float64), - tcp_s2c_rtx_pkts_ratio Nullable(Float64), - tcp_rtx_pkts_ratio Nullable(Float64), - avg_tcp_rtt_ms Nullable(Float64), - avg_http_response_latency_ms Nullable(Float64), - avg_ssl_handshake_latency_ms Nullable(Float64) -) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_local', rand()); - - CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app_local ON CLUSTER ck_cluster ( subscriber_id String, app String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), stat_time Int64, sent_pkts Int64, sent_bytes Int64, @@ -682,11 +576,17 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app_local ON avg_tcp_rtt_ms Nullable(Float64), avg_http_response_latency_ms Nullable(Float64), avg_ssl_handshake_latency_ms Nullable(Float64) -) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192; +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id,app) SETTINGS index_granularity = 8192; CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTER ck_cluster ( subscriber_id String, app String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), stat_time Int64, sent_pkts Int64, sent_bytes Int64, @@ -721,6 +621,12 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTER ck_query ( subscriber_id String, app String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), stat_time Int64, sent_pkts Int64, sent_bytes Int64, @@ -749,5 +655,5 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE tcp_rtx_pkts_ratio Nullable(Float64), avg_tcp_rtt_ms Nullable(Float64), avg_http_response_latency_ms Nullable(Float64), - avg_ssl_handshake_latency_ms Nullable(Float64) + avg_ssl_handshake_latency_ms Nullable(Float64) ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand()); diff --git a/CN发布版本更新记录/CN-23.12/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_subscriber_app_cn.sh b/CN发布版本更新记录/CN-23.12/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_subscriber_app_cn.sh new file mode 100644 index 0000000..42d34c0 --- /dev/null +++ b/CN发布版本更新记录/CN-23.12/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_subscriber_app_cn.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_metric_subscriber_app_cn +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=5 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/CN发布版本更新记录/CN-23.12/gohangout/conf/k2ck_metric_subscriber_app_cn.yml b/CN发布版本更新记录/CN-23.12/gohangout/conf/k2ck_metric_subscriber_app_cn.yml new file mode 100644 index 0000000..0d1b18d --- /dev/null +++ b/CN发布版本更新记录/CN-23.12/gohangout/conf/k2ck_metric_subscriber_app_cn.yml @@ -0,0 +1,28 @@ +inputs: + - Kafka: + topic: + METRIC-SUBSCRIBER-APP: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "192.168.44.55:9092" + group.id: metric_subscriber_app_1 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'cyber_narrator_galaxy.metric_subscriber_app_local' + username: 'default' + password: 'ceiec2019' + hosts: + - 'tcp://192.168.44.55:9001' + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/CN发布版本更新记录/CN-23.12/gohangout/start_all.sh b/CN发布版本更新记录/CN-23.12/gohangout/start_all.sh new file mode 100644 index 0000000..afe14f9 --- /dev/null +++ b/CN发布版本更新记录/CN-23.12/gohangout/start_all.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +STARTDIR=$(cd $(dirname $0); pwd) +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_subscriber_app_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_security_event_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_region_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_asn_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_application_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_domain_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_protocol_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_link_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qname_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_server_ip_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qtype_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rcode_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_a_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_aaaa_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_cname_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh $1 > /dev/null 2>&1 & +$STARTDIR/bin/ghoStart/start_gohangout_k2ck_cn_record.sh $1 diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql index 5c93f40..7a8d7a0 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql @@ -1134,121 +1134,15 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_link ON CLUSTER ck_query ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_link_local', rand()); -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_local ON CLUSTER ck_cluster ( - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - stat_time Int64, - sent_pkts Int64, - sent_bytes Int64, - received_pkts Int64, - received_bytes Int64, - sessions Int64, - traffic_inbound_byte Int64, - traffic_inbound_pkt Int64, - traffic_outbound_byte Int64, - traffic_outbound_pkt Int64, - traffic_internal_byte Int64, - traffic_internal_pkt Int64, - traffic_through_byte Int64, - traffic_through_pkt Int64, - tcp_c2s_lost_bytes_ratio Nullable(Float64), - tcp_s2c_lost_bytes_ratio Nullable(Float64), - tcp_lost_bytes_ratio Nullable(Float64), - tcp_c2s_o3_pkts_ratio Nullable(Float64), - tcp_s2c_o3_pkts_ratio Nullable(Float64), - tcp_o3_pkts_ratio Nullable(Float64), - tcp_c2s_rtx_bytes_ratio Nullable(Float64), - tcp_s2c_rtx_bytes_ratio Nullable(Float64), - tcp_rtx_bytes_ratio Nullable(Float64), - tcp_c2s_rtx_pkts_ratio Nullable(Float64), - tcp_s2c_rtx_pkts_ratio Nullable(Float64), - tcp_rtx_pkts_ratio Nullable(Float64), - avg_tcp_rtt_ms Nullable(Float64), - avg_http_response_latency_ms Nullable(Float64), - avg_ssl_handshake_latency_ms Nullable(Float64) -) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192; - -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber ON CLUSTER ck_cluster ( - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - stat_time Int64, - sent_pkts Int64, - sent_bytes Int64, - received_pkts Int64, - received_bytes Int64, - sessions Int64, - traffic_inbound_byte Int64, - traffic_inbound_pkt Int64, - traffic_outbound_byte Int64, - traffic_outbound_pkt Int64, - traffic_internal_byte Int64, - traffic_internal_pkt Int64, - traffic_through_byte Int64, - traffic_through_pkt Int64, - tcp_c2s_lost_bytes_ratio Nullable(Float64), - tcp_s2c_lost_bytes_ratio Nullable(Float64), - tcp_lost_bytes_ratio Nullable(Float64), - tcp_c2s_o3_pkts_ratio Nullable(Float64), - tcp_s2c_o3_pkts_ratio Nullable(Float64), - tcp_o3_pkts_ratio Nullable(Float64), - tcp_c2s_rtx_bytes_ratio Nullable(Float64), - tcp_s2c_rtx_bytes_ratio Nullable(Float64), - tcp_rtx_bytes_ratio Nullable(Float64), - tcp_c2s_rtx_pkts_ratio Nullable(Float64), - tcp_s2c_rtx_pkts_ratio Nullable(Float64), - tcp_rtx_pkts_ratio Nullable(Float64), - avg_tcp_rtt_ms Nullable(Float64), - avg_http_response_latency_ms Nullable(Float64), - avg_ssl_handshake_latency_ms Nullable(Float64) -) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_local', rand()); - -CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber ON CLUSTER ck_query ( - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - stat_time Int64, - sent_pkts Int64, - sent_bytes Int64, - received_pkts Int64, - received_bytes Int64, - sessions Int64, - traffic_inbound_byte Int64, - traffic_inbound_pkt Int64, - traffic_outbound_byte Int64, - traffic_outbound_pkt Int64, - traffic_internal_byte Int64, - traffic_internal_pkt Int64, - traffic_through_byte Int64, - traffic_through_pkt Int64, - tcp_c2s_lost_bytes_ratio Nullable(Float64), - tcp_s2c_lost_bytes_ratio Nullable(Float64), - tcp_lost_bytes_ratio Nullable(Float64), - tcp_c2s_o3_pkts_ratio Nullable(Float64), - tcp_s2c_o3_pkts_ratio Nullable(Float64), - tcp_o3_pkts_ratio Nullable(Float64), - tcp_c2s_rtx_bytes_ratio Nullable(Float64), - tcp_s2c_rtx_bytes_ratio Nullable(Float64), - tcp_rtx_bytes_ratio Nullable(Float64), - tcp_c2s_rtx_pkts_ratio Nullable(Float64), - tcp_s2c_rtx_pkts_ratio Nullable(Float64), - tcp_rtx_pkts_ratio Nullable(Float64), - avg_tcp_rtt_ms Nullable(Float64), - avg_http_response_latency_ms Nullable(Float64), - avg_ssl_handshake_latency_ms Nullable(Float64) -) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_local', rand()); - - CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app_local ON CLUSTER ck_cluster ( subscriber_id String, app String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), stat_time Int64, sent_pkts Int64, sent_bytes Int64, @@ -1278,11 +1172,17 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app_local ON avg_tcp_rtt_ms Nullable(Float64), avg_http_response_latency_ms Nullable(Float64), avg_ssl_handshake_latency_ms Nullable(Float64) -) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id) SETTINGS index_granularity = 8192; +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (stat_time,subscriber_id,app) SETTINGS index_granularity = 8192; CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTER ck_cluster ( subscriber_id String, app String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), stat_time Int64, sent_pkts Int64, sent_bytes Int64, @@ -1317,6 +1217,12 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTER ck_query ( subscriber_id String, app String, + imei String, + imsi String, + phone_number String, + apn String, + subscriber_longitude Nullable(Float64), + subscriber_latitude Nullable(Float64), stat_time Int64, sent_pkts Int64, sent_bytes Int64, @@ -1345,7 +1251,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE tcp_rtx_pkts_ratio Nullable(Float64), avg_tcp_rtt_ms Nullable(Float64), avg_http_response_latency_ms Nullable(Float64), - avg_ssl_handshake_latency_ms Nullable(Float64) + avg_ssl_handshake_latency_ms Nullable(Float64) ) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand());