diff --git a/TSG-21.08/clickhouse/create_dos_tables.bash b/TSG-21.08/clickhouse/create_dos_tables.bash new file mode 100644 index 0000000..ef2dfba --- /dev/null +++ b/TSG-21.08/clickhouse/create_dos_tables.bash @@ -0,0 +1,34 @@ + + +CREATE TABLE tsg_galaxy_v3.dos_event_log_local on cluster ck_cluster( +log_id UInt64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(start_time)) ORDER BY (log_id, start_time, destination_ip) SETTINGS index_granularity = 8192; + + +CREATE TABLE tsg_galaxy_v3.dos_event_log on cluster ck_query( + log_id UInt64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) ENGINE = Distributed('ck_cluster', 'tsg_galaxy_v3', 'dos_event_log_local', rand()); \ No newline at end of file diff --git a/TSG-21.08/druid/traffic_destination_ip_metrics_log.json b/TSG-21.08/druid/traffic_destination_ip_metrics_log.json new file mode 100644 index 0000000..d57893b --- /dev/null +++ b/TSG-21.08/druid/traffic_destination_ip_metrics_log.json @@ -0,0 +1,55 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_top_destination_ip_metrics_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "sketch_start_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "common_sled_ip", + "destination_ip", + "common_data_center", + "attack_type", + { "type": "long", "name": "partition_num" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"}, + { "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" }, + { "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" }, + { "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT10M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[] + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-DESTINATION-IP-METRICS-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "192.168.44.12:9092" + } + } +} diff --git a/TSG-21.08/druid/traffic_summary_log.json b/TSG-21.08/druid/traffic_summary_log.json new file mode 100644 index 0000000..968986e --- /dev/null +++ b/TSG-21.08/druid/traffic_summary_log.json @@ -0,0 +1,72 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_summary_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "isp", + "data_center", + "schema_type", + { "name": "entrance_id", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" }, + { "type" : "longSum", "name" : "sessions", "fieldName" : "common_sessions" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" }, + { "type" : "longSum", "name" : "one_sided_connections", "fieldName" : "one_sided_connections" }, + { "type" : "longSum", "name" : "uncategorized_bytes", "fieldName" : "uncategorized_bytes" }, + { "type" : "longSum", "name" : "fragmentation_packets", "fieldName" : "fragmentation_packets" }, + { "type" : "longSum", "name" : "sequence_gap_loss", "fieldName" : "sequence_gap_loss" }, + { "type" : "longSum", "name" : "unorder_packets", "fieldName" : "unorder_packets" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression","name": "isp","expression": "common_isp"}, + {"type": "expression","name": "data_center","expression": "common_data_center"}, + {"type": "expression","name": "entrance_id","expression": "common_entrance_id"}, + {"type": "expression","name": "schema_type","expression": "common_schema_type"}, + {"type": "expression", "name": "one_sided_connections", "expression": "if(common_stream_dir!=3,common_sessions,0)"}, + {"type": "expression", "name": "uncategorized_bytes", "expression": "if(\"common_l7_protocol\"=='UNCATEGORIZED',\"common_c2s_byte_num\"+\"common_s2c_byte_num\",0)"}, + {"type": "expression","name": "fragmentation_packets","expression": "\"common_c2s_ipfrag_num\"+\"common_s2c_ipfrag_num\""}, + {"type": "expression","name": "sequence_gap_loss","expression": "\"common_c2s_tcp_lostlen\"+\"common_s2c_tcp_lostlen\""}, + {"type": "expression","name": "unorder_packets","expression": "\"common_c2s_tcp_unorder_num\"+\"common_s2c_tcp_unorder_num\""} + ] + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 20000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "CONNECTION-RECORD-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap:9092", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/TSG-21.08/gohangout/bin/ghoStart/start_gohangout_k2ck_dos_event_log_tsgv3.sh b/TSG-21.08/gohangout/bin/ghoStart/start_gohangout_k2ck_dos_event_log_tsgv3.sh new file mode 100644 index 0000000..b8b0074 --- /dev/null +++ b/TSG-21.08/gohangout/bin/ghoStart/start_gohangout_k2ck_dos_event_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_dos_event_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=1 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/TSG-21.08/gohangout/conf/k2ck_dos_event_log_tsgv3.yml b/TSG-21.08/gohangout/conf/k2ck_dos_event_log_tsgv3.yml new file mode 100644 index 0000000..d3bf701 --- /dev/null +++ b/TSG-21.08/gohangout/conf/k2ck_dos_event_log_tsgv3.yml @@ -0,0 +1,29 @@ +inputs: + - Kafka: + topic: + DOS-EVENT-LOG: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "ip:9092" + group.id: gohangout_dos_event_log_tsgv3_1 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.dos_event_log_local' + username: 'default' + password: 'ceiec2019' + conn_max_life_time: 60 + hosts: + - 'tcp://ip:9001' + bulk_actions: 100000 + flush_interval: 10 + concurrent: 2 + conn_max_life_time: 60 diff --git a/TSG-21.08/gohangout/start_all.sh b/TSG-21.08/gohangout/start_all.sh new file mode 100644 index 0000000..c84125d --- /dev/null +++ b/TSG-21.08/gohangout/start_all.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +STARTDIR=$(cd $(dirname $0); pwd) + +#docker模式 +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_active_defence_event_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_connection_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_proxy_event_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_radius_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_radius_onff_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_sys_packet_capture_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_voip_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_gtpc_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_live_session_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_transaction_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_dos_event_log_tsgv3.sh $1 > /dev/null 2>&1 & +$STARTDIR/bin/ghoStart/start_gohangout_k2ck_security_event_log_tsgv3.sh $1