From 9e128e2334dd4e8f945f2da2aad4e7a19bc410ac Mon Sep 17 00:00:00 2001 From: houjinchuan Date: Wed, 16 Aug 2023 14:06:55 +0800 Subject: [PATCH] =?UTF-8?q?CN=2023.09=20=E6=B7=BB=E5=8A=A0gohangout?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...t_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh | 51 +++++++++++++++++++ .../start_gohangout_k2ck_metric_relation_cn.sh | 51 +++++++++++++++++++ .../conf/k2ck_metric_ip_dynamic_attribute_cn.yml | 28 ++++++++++ .../gohangout/conf/k2ck_metric_relation_cn.yml | 28 ++++++++++ .../CN-23.09/gohangout/start_all.sh | 21 ++++++++ 5 files changed, 179 insertions(+) create mode 100644 CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh create mode 100644 CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh create mode 100644 CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_ip_dynamic_attribute_cn.yml create mode 100644 CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_relation_cn.yml create mode 100644 CN发布版本更新记录/CN-23.09/gohangout/start_all.sh diff --git a/CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh b/CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh new file mode 100644 index 0000000..612bf12 --- /dev/null +++ b/CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_metric_ip_dynamic_attribute_cn +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=5 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh b/CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh new file mode 100644 index 0000000..2c67625 --- /dev/null +++ b/CN发布版本更新记录/CN-23.09/gohangout/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_metric_relation_cn +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=5 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_ip_dynamic_attribute_cn.yml b/CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_ip_dynamic_attribute_cn.yml new file mode 100644 index 0000000..799fdce --- /dev/null +++ b/CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_ip_dynamic_attribute_cn.yml @@ -0,0 +1,28 @@ +inputs: + - Kafka: + topic: + METRIC-IP-DYNAMIC-ATTRIBUTE: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "192.168.44.55:9092" + group.id: tsg-metric_ip_dynamic_attribute_cn + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'cyber_narrator_galaxy.metric_ip_dynamic_attribute_local' + username: 'default' + password: 'ceiec2019' + hosts: + - 'tcp://192.168.44.55:9001' + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_relation_cn.yml b/CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_relation_cn.yml new file mode 100644 index 0000000..99e8e85 --- /dev/null +++ b/CN发布版本更新记录/CN-23.09/gohangout/conf/k2ck_metric_relation_cn.yml @@ -0,0 +1,28 @@ +inputs: + - Kafka: + topic: + METRIC-RELATION: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "192.168.44.55:9092" + group.id: tsg-metric_relation_cn + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'cyber_narrator_galaxy.metric_relation_local' + username: 'default' + password: 'ceiec2019' + hosts: + - 'tcp://192.168.44.55:9001' + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/CN发布版本更新记录/CN-23.09/gohangout/start_all.sh b/CN发布版本更新记录/CN-23.09/gohangout/start_all.sh new file mode 100644 index 0000000..4d6d42f --- /dev/null +++ b/CN发布版本更新记录/CN-23.09/gohangout/start_all.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +STARTDIR=$(cd $(dirname $0); pwd) + +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_region_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_asn_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_application_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_domain_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_protocol_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_link_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qname_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_server_ip_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_qtype_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rcode_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_a_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_aaaa_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_dns_rr_cname_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_relation_cn.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_metric_ip_dynamic_attribute_cn.sh $1 > /dev/null 2>&1 & +$STARTDIR/bin/ghoStart/start_gohangout_k2ck_cn_record.sh $1 \ No newline at end of file