gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'knowledge' of https://git.mesalab.cn/bigdata/tsg/flink-dos-detection into tsg-22.11

Browse Source
This commit is contained in:
wanglihui
2022-12-06 19:11:30 +08:00
parent 76c9247bb9
commit 78435d54ea
5 changed files with 53 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/main/java/com/zdjizhi/etl/DosDetection.java
View File

@@ -10,7 +10,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction;
import org.apache.flink.util.Collector;
import java.math.BigDecimal;
@@ -23,9 +23,9 @@ import java.util.concurrent.TimeUnit;
/**
* @author wlh
*/
public class DosDetection extends ProcessFunction<DosSketchLog, DosEventLog> {
public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<String, byte[]>, DosEventLog> {
// private static final Logger logger = LoggerFactory.getLogger(DosDetection.class);
// private static final Logger logger = LoggerFactory.getLogger(DosDetection.class);
private static final Log logger = LogFactory.get();
private static Map<String, Map<String, DosBaselineThreshold>> baselineMap = new HashMap<>();
private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance();
@@ -47,6 +47,12 @@ public class DosDetection extends ProcessFunction<DosSketchLog, DosEventLog> {
ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(2,
new BasicThreadFactory.Builder().namingPattern("Dos-Detection-%d").daemon(true).build());
try {
super.open(parameters);
logger.info("begin init");
IpUtils.loadIpLook();
logger.info("init over");
executorService.scheduleAtFixedRate(() -> thresholdRangeMap = ParseStaticThreshold.createStaticThreshold(), 0,
CommonConfig.STATIC_THRESHOLD_SCHEDULE_MINUTES, TimeUnit.MINUTES);
@@ -59,7 +65,7 @@ public class DosDetection extends ProcessFunction<DosSketchLog, DosEventLog> {
}
@Override
public void processElement(DosSketchLog value, Context ctx, Collector<DosEventLog> out) {
public void processElement(DosSketchLog value, ReadOnlyContext ctx, Collector<DosEventLog> out) {
ArrayList<DosEventLog> finalResults = new ArrayList<>();
try {
String destinationIp = value.getDestination_ip();
@@ -98,6 +104,11 @@ public class DosDetection extends ProcessFunction<DosSketchLog, DosEventLog> {
}
}
@Override
public void processBroadcastElement(Map<String, byte[]> value, Context ctx, Collector<DosEventLog> out) throws Exception {
IpUtils.updateIpLook(value);
}
private DosEventLog getDosEventLogBySensitivityThreshold(DosSketchLog value) {
long sketchSessions = value.getSketch_sessions();
Integer staticSensitivityThreshold = NacosUtils.getIntProperty("static.sensitivity.threshold");