1：新增dos相关任务

2：修复druid-TSG-7519问题
2021-08-23 16:39:48 +08:00
parent 9e2fbcd9da
commit 5b1cc5f2ba
6 changed files with 258 additions and 0 deletions
--- a/TSG-21.08/clickhouse/create_dos_tables.bash
+++ b/TSG-21.08/clickhouse/create_dos_tables.bash
@@ -0,0 +1,34 @@
+
+
+CREATE TABLE tsg_galaxy_v3.dos_event_log_local on cluster ck_cluster(
+log_id UInt64,
+ start_time Int64,
+ end_time Int64,
+ attack_type String,
+ severity String,
+ conditions String,
+ destination_ip String,
+ destination_country String,
+ source_ip_list String,
+ source_country_list String,
+ session_rate Int64,
+ packet_rate Int64,
+ bit_rate Int64
+) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(start_time)) ORDER BY (log_id, start_time, destination_ip) SETTINGS index_granularity = 8192;
+
+
+CREATE TABLE tsg_galaxy_v3.dos_event_log on cluster ck_query(
+ log_id UInt64,
+ start_time Int64,
+ end_time Int64,
+ attack_type String,
+ severity String,
+ conditions String,
+ destination_ip String,
+ destination_country String,
+ source_ip_list String,
+ source_country_list String,
+ session_rate Int64,
+ packet_rate Int64,
+ bit_rate Int64
+) ENGINE = Distributed('ck_cluster', 'tsg_galaxy_v3', 'dos_event_log_local', rand());
--- a/TSG-21.08/druid/traffic_destination_ip_metrics_log.json
+++ b/TSG-21.08/druid/traffic_destination_ip_metrics_log.json
@@ -0,0 +1,55 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "traffic_top_destination_ip_metrics_log",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "sketch_start_time",
+          "format": "posix"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+	  "common_sled_ip",
+	  "destination_ip",
+	  "common_data_center",
+	  "attack_type",
+	  { "type": "long", "name": "partition_num" }
+          ]
+        }
+      }
+    },
+    "metricsSpec" : [
+	{ "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"},
+        { "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" },
+        { "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" },
+	{ "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT10M", "timeZone": "Asia/Hong_Kong"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[]
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-DESTINATION-IP-METRICS-LOG",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "192.168.44.12:9092"
+    }
+  }
+}
--- a/TSG-21.08/druid/traffic_summary_log.json
+++ b/TSG-21.08/druid/traffic_summary_log.json
@@ -0,0 +1,72 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "traffic_summary_log",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "common_recv_time",
+          "format": "posix"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+	   "isp",
+	   "data_center",
+           "schema_type",
+           { "name": "entrance_id", "type": "long" }
+          ]
+        }
+      }
+    },
+    "metricsSpec" : [
+	{ "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" },
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "common_sessions" },
+        { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" },
+        { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" },
+        { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" },
+        { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" },
+        { "type" : "longSum", "name" : "one_sided_connections", "fieldName" : "one_sided_connections" },
+        { "type" : "longSum", "name" : "uncategorized_bytes", "fieldName" : "uncategorized_bytes" },
+        { "type" : "longSum", "name" : "fragmentation_packets", "fieldName" : "fragmentation_packets" },
+        { "type" : "longSum", "name" : "sequence_gap_loss", "fieldName" : "sequence_gap_loss" },
+        { "type" : "longSum", "name" : "unorder_packets", "fieldName" : "unorder_packets" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT15S", "timeZone": "Asia/Hong_Kong"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression","name": "isp","expression": "common_isp"},
+        {"type": "expression","name": "data_center","expression": "common_data_center"},
+        {"type": "expression","name": "entrance_id","expression": "common_entrance_id"},
+        {"type": "expression","name": "schema_type","expression": "common_schema_type"},
+	{"type": "expression", "name": "one_sided_connections", "expression": "if(common_stream_dir!=3,common_sessions,0)"},
+        {"type": "expression", "name": "uncategorized_bytes", "expression": "if(\"common_l7_protocol\"=='UNCATEGORIZED',\"common_c2s_byte_num\"+\"common_s2c_byte_num\",0)"},
+        {"type": "expression","name": "fragmentation_packets","expression": "\"common_c2s_ipfrag_num\"+\"common_s2c_ipfrag_num\""},
+        {"type": "expression","name": "sequence_gap_loss","expression": "\"common_c2s_tcp_lostlen\"+\"common_s2c_tcp_lostlen\""},
+        {"type": "expression","name": "unorder_packets","expression": "\"common_c2s_tcp_unorder_num\"+\"common_s2c_tcp_unorder_num\""}
+      ]
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "maxRowsPerSegment": 20000000,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "CONNECTION-RECORD-COMPLETED-LOG",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap:9092",
+      "group.id": "kafka-indexing-service"
+    }
+  }
+}
--- a/TSG-21.08/gohangout/bin/ghoStart/start_gohangout_k2ck_dos_event_log_tsgv3.sh
+++ b/TSG-21.08/gohangout/bin/ghoStart/start_gohangout_k2ck_dos_event_log_tsgv3.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+#gohangout启动文件
+
+#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下
+YML_NAME=k2ck_dos_event_log_tsgv3
+#gohangout的二进制启动文件路径
+BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
+#日志级别,1,5,10,数字越大日志越详细
+LOG_LV=3
+#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程
+THREAD_SUM=1
+#进程总数
+PROCESS_SUM=$1
+
+if [ ! -d "$BASE_DIR/logs" ]; then
+        mkdir -p $BASE_DIR/logs
+fi
+
+echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+id=0
+logid=0
+while true ; do
+    NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l`
+    pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
+    time_stamp=$(date +%Y%m%d%H%M%S)
+    if [ "${NUM}" -lt ${PROCESS_SUM} ];then
+	$BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 &
+    echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+    ((logid++))
+    ((id++))
+     if [ ${logid} -gt ${PROCESS_SUM} ];then
+      logid=0
+      pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}')
+        for pid in $pids
+        do
+         kill -9  $pid
+        done
+        echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log
+        sleep 30
+     fi
+    #大于设置进程数,杀掉所有进程,重启
+    elif [ "${NUM}" -gt ${PROCESS_SUM} ];then
+        for pid in $pids
+        do
+         kill -9  $pid
+        done
+    id=0
+    fi
+        sleep 1
+done
--- a/TSG-21.08/gohangout/conf/k2ck_dos_event_log_tsgv3.yml
+++ b/TSG-21.08/gohangout/conf/k2ck_dos_event_log_tsgv3.yml
@@ -0,0 +1,29 @@
+inputs:
+    - Kafka:
+        topic:
+            DOS-EVENT-LOG: 1
+        #assign:
+        #  weblog: [0,9]
+        codec: json
+        consumer_settings:
+            bootstrap.servers: "ip:9092"
+            group.id: gohangout_dos_event_log_tsgv3_1
+            max.partition.fetch.bytes: '10485760'
+            auto.commit.interval.ms: '5000'
+            # from.beginning: 'true'
+            # sasl.mechanism: PLAIN
+            # sasl.user: admin
+            # sasl.password: admin-secret
+
+outputs:
+    - Clickhouse:
+        table: 'tsg_galaxy_v3.dos_event_log_local'
+        username: 'default'
+        password: 'ceiec2019'
+        conn_max_life_time: 60
+        hosts:
+        - 'tcp://ip:9001'
+        bulk_actions: 100000
+        flush_interval: 10
+        concurrent: 2
+        conn_max_life_time: 60
--- a/TSG-21.08/gohangout/start_all.sh
+++ b/TSG-21.08/gohangout/start_all.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+STARTDIR=$(cd $(dirname $0); pwd)
+
+#docker模式
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_active_defence_event_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_connection_record_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_proxy_event_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_radius_record_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_radius_onff_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_sys_packet_capture_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_voip_record_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_gtpc_record_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_live_session_record_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_transaction_record_log_tsgv3.sh $1 > /dev/null 2>&1 &
+nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_dos_event_log_tsgv3.sh $1 > /dev/null 2>&1 &
+$STARTDIR/bin/ghoStart/start_gohangout_k2ck_security_event_log_tsgv3.sh $1