gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加异常检测信息，README.md文件

Browse Source 
修复maven打包携带META信息
This commit is contained in:
wanglihui
2021-07-30 10:55:01 +08:00
parent 5816978f56
commit 76e078e073
9 changed files with 90 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/com/zdjizhi/etl/EtlProcessFunction.java
View File

@@ -4,6 +4,7 @@ import com.zdjizhi.common.CommonConfig;
import com.zdjizhi.common.DosSketchLog;
import org.apache.commons.lang.StringUtils;
import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.api.java.tuple.Tuple6;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;
@@ -28,7 +29,7 @@ public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosS
try {
if (middleResult != null){
out.collect(middleResult);
logger.debug("获取中间聚合结果：{}",middleResult.toString());
logger.info("获取中间聚合结果：{}",middleResult.toString());
context.output(outputTag,TrafficServerIpMetrics.getOutputMetric(keys, middleResult));
}
}catch (Exception e){
@@ -39,13 +40,15 @@ public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosS
private DosSketchLog getMiddleResult(Tuple4<String,String, String, String> keys,Iterable<DosSketchLog> elements){
DosSketchLog midResuleLog = new DosSketchLog();
Tuple4<Long, Long, Long,String> values = sketchAggregate(elements);
Tuple6<Long, Long, Long,String,Long,Long> values = sketchAggregate(elements);
try {
if (values != null){
midResuleLog.setCommon_sled_ip(keys.f0);
midResuleLog.setCommon_data_center(keys.f1);
midResuleLog.setDestination_ip(keys.f3);
midResuleLog.setAttack_type(keys.f2);
midResuleLog.setSketch_start_time(values.f4);
midResuleLog.setSketch_duration(values.f5);
midResuleLog.setSource_ip(values.f3);
midResuleLog.setSketch_sessions(values.f0);
midResuleLog.setSketch_packets(values.f1);
@@ -58,24 +61,28 @@ public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosS
return null;
}
private Tuple4<Long, Long, Long,String> sketchAggregate(Iterable<DosSketchLog> elements){
private Tuple6<Long, Long, Long,String,Long,Long> sketchAggregate(Iterable<DosSketchLog> elements){
int cnt = 1;
long sessions = 0;
long packets = 0 ;
long bytes = 0;
long startTime = 0;
long duration = 0;
HashSet<String> sourceIpSet = new HashSet<>();
try {
for (DosSketchLog newSketchLog : elements){
sessions += newSketchLog.getSketch_sessions();
packets += newSketchLog.getSketch_packets();
bytes += newSketchLog.getSketch_bytes();
startTime = newSketchLog.getSketch_start_time();
duration = newSketchLog.getSketch_duration();
cnt += 1;
if (sourceIpSet.size() < CommonConfig.SOURCE_IP_LIST_LIMIT){
sourceIpSet.add(newSketchLog.getSource_ip());
}
}
String sourceIpList = StringUtils.join(sourceIpSet, ",");
return Tuple4.of(sessions/cnt,packets/cnt,bytes/cnt,sourceIpList);
return Tuple6.of(sessions/cnt,packets/cnt,bytes/cnt,sourceIpList,startTime,duration);
}catch (Exception e){
logger.error("聚合中间结果集失败 {}",e);
}