CN 23.09 添加gohangout任务

CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+#gohangout启动文件
+
+#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下
+YML_NAME=k2ck_metric_ip_dynamic_attribute_cn
+#gohangout的二进制启动文件路径
+BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
+#日志级别,1,5,10,数字越大日志越详细
+LOG_LV=5
+#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程
+THREAD_SUM=3
+#进程总数
+PROCESS_SUM=$1
+
+if [ ! -d "$BASE_DIR/logs" ]; then
+        mkdir -p $BASE_DIR/logs
+fi
+
+echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+id=0
+logid=0
+while true ; do
+    NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l`
+    pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
+    time_stamp=$(date +%Y%m%d%H%M%S)
+    if [ "${NUM}" -lt ${PROCESS_SUM} ];then
+	$BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 &
+    echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+    ((logid++))
+    ((id++))
+     if [ ${logid} -gt ${PROCESS_SUM} ];then
+      logid=0
+      pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
+        for pid in $pids
+        do
+         kill -9  $pid
+        done
+        echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+        sleep 30
+     fi
+    #大于设置进程数,杀掉所有进程,重启
+    elif [ "${NUM}" -gt ${PROCESS_SUM} ];then
+        for pid in $pids
+        do
+         kill -9  $pid
+        done
+    id=0
+    fi
+        sleep 1
+done

CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+#gohangout启动文件
+
+#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下
+YML_NAME=k2ck_metric_relation_cn
+#gohangout的二进制启动文件路径
+BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
+#日志级别,1,5,10,数字越大日志越详细
+LOG_LV=5
+#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程
+THREAD_SUM=3
+#进程总数
+PROCESS_SUM=$1
+
+if [ ! -d "$BASE_DIR/logs" ]; then
+        mkdir -p $BASE_DIR/logs
+fi
+
+echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+id=0
+logid=0
+while true ; do
+    NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l`
+    pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
+    time_stamp=$(date +%Y%m%d%H%M%S)
+    if [ "${NUM}" -lt ${PROCESS_SUM} ];then
+	$BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 &
+    echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+    ((logid++))
+    ((id++))
+     if [ ${logid} -gt ${PROCESS_SUM} ];then
+      logid=0
+      pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
+        for pid in $pids
+        do
+         kill -9  $pid
+        done
+        echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+        sleep 30
+     fi
+    #大于设置进程数,杀掉所有进程,重启
+    elif [ "${NUM}" -gt ${PROCESS_SUM} ];then
+        for pid in $pids
+        do
+         kill -9  $pid
+        done
+    id=0
+    fi
+        sleep 1
+done

CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_ip_dynamic_attribute_cn.yml

@@ -0,0 +1,28 @@
+inputs:
+    - Kafka:
+        topic:
+            METRIC-IP-DYNAMIC-ATTRIBUTE: 1
+        #assign:
+        #  weblog: [0,9]
+        codec: json
+        consumer_settings:
+            bootstrap.servers: "192.168.44.55:9092"
+            group.id: tsg-metric_ip_dynamic_attribute_cn
+            max.partition.fetch.bytes: '10485760'
+            auto.commit.interval.ms: '5000'
+            # from.beginning: 'true'
+            # sasl.mechanism: PLAIN
+            # sasl.user: admin
+            # sasl.password: admin-secret
+
+outputs:
+    - Clickhouse:
+        table: 'cyber_narrator_galaxy.metric_ip_dynamic_attribute_local'
+        username: 'default'
+        password: 'ceiec2019'
+        hosts:
+        - 'tcp://192.168.44.55:9001'
+        bulk_actions: 100000
+        flush_interval: 30
+        concurrent: 2
+        conn_max_life_time: 60

CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_relation_cn.yml

@@ -0,0 +1,28 @@
+inputs:
+    - Kafka:
+        topic:
+            METRIC-RELATION: 1
+        #assign:
+        #  weblog: [0,9]
+        codec: json
+        consumer_settings:
+            bootstrap.servers: "192.168.44.55:9092"
+            group.id: tsg-metric_relation_cn
+            max.partition.fetch.bytes: '10485760'
+            auto.commit.interval.ms: '5000'
+            # from.beginning: 'true'
+            # sasl.mechanism: PLAIN
+            # sasl.user: admin
+            # sasl.password: admin-secret
+
+outputs:
+    - Clickhouse:
+        table: 'cyber_narrator_galaxy.metric_relation_local'
+        username: 'default'
+        password: 'ceiec2019'
+        hosts:
+        - 'tcp://192.168.44.55:9001'
+        bulk_actions: 100000
+        flush_interval: 30
+        concurrent: 2
+        conn_max_life_time: 60

CN发布版本更新记录/CN-23.09/gohangout/start_all.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+STARTDIR=$(cd $(dirname $0); pwd)
+
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_region_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_asn_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_application_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_domain_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_protocol_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_link_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qname_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_server_ip_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qtype_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rcode_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_a_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_aaaa_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_cname_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh $1 > /dev/null 2>&1 &
+$STARTDIR/bin/ghoStart/start_gohangout_k2ck_cn_record.sh $1