gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增根据静态阈值判定dos攻击逻辑

Browse Source 
新增定时器，定时获取静态阈值与baseline
This commit is contained in:
wanglihui
2021-08-24 16:35:31 +08:00
parent 55af33b508
commit b4f919647a
7 changed files with 173 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
src/main/java/com/zdjizhi/etl/DosDetection.java
View File

@@ -10,6 +10,7 @@ import com.zdjizhi.utils.SnowflakeId;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressString;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
@@ -20,6 +21,9 @@ import org.slf4j.LoggerFactory;
import java.text.NumberFormat;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* @author wlh
@@ -27,15 +31,28 @@ import java.util.*;
public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
private static final Logger logger = LoggerFactory.getLogger(DosDetection.class);
private static Map<String, Map<String, Tuple2<ArrayList<Integer>, Integer>>> baselineMap;
private static Map<String, Map<String, Tuple2<ArrayList<Integer>, Integer>>> baselineMap = new HashMap<>();
private final static int BASELINE_SIZE = 144;
private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance();
private TreeRangeMap<IPAddress, Map<String, DosDetectionThreshold>> thresholdRangeMap;
@Override
public void open(Configuration parameters) {
baselineMap = HbaseUtils.baselineMap;
thresholdRangeMap = ParseStaticThreshold.createStaticThreshold();
ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(2,
new BasicThreadFactory.Builder().namingPattern("Dos-Detection-%d").daemon(true).build());
try {
executorService.scheduleAtFixedRate(() -> {
//do something
thresholdRangeMap = ParseStaticThreshold.createStaticThreshold();
}, 0, CommonConfig.STATIC_THRESHOLD_SCHEDULE_MINUTES, TimeUnit.MINUTES);
executorService.scheduleAtFixedRate(() -> {
//do something
baselineMap = HbaseUtils.readFromHbase();
}, 0, CommonConfig.BASELINE_THRESHOLD_SCHEDULE_DAYS, TimeUnit.DAYS);
} catch (Exception e) {
logger.error("定时器任务执行失败", e);
}
PERCENT_INSTANCE.setMinimumFractionDigits(2);
}
@@ -48,72 +65,90 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
IPAddress destinationIpAddress = new IPAddressString(destinationIp).getAddress();
Map<String, DosDetectionThreshold> thresholdMap = thresholdRangeMap.get(destinationIpAddress);
logger.debug("当前判断IP：{}, 类型: {}", destinationIp, attackType);
if (baselineMap.containsKey(destinationIp) && thresholdMap == null) {
finalResult = getDosEventLogByBaseline(value, destinationIp, attackType);
} else if (!baselineMap.containsKey(destinationIp) && thresholdMap != null) {
finalResult = getDosEventLogByStaticThreshold(value,thresholdMap);
}else if (baselineMap.containsKey(destinationIp) && thresholdMap != null){
DosEventLog eventLogByBaseline = getDosEventLogByBaseline(value, destinationIp, attackType);
DosEventLog eventLogByStaticThreshold = getDosEventLogByStaticThreshold(value, thresholdMap);
finalResult = mergeFinalResult(eventLogByBaseline,eventLogByStaticThreshold);
}else {
if (baselineMap != null && baselineMap.containsKey(destinationIp) && thresholdMap == null) {
finalResult = getDosEventLogByBaseline(value, destinationIp, attackType).f1;
} else if (baselineMap != null && !baselineMap.containsKey(destinationIp) && thresholdMap != null) {
finalResult = getDosEventLogByStaticThreshold(value, thresholdMap).f1;
} else if (baselineMap != null && baselineMap.containsKey(destinationIp) && thresholdMap != null) {
Tuple2<Severity, DosEventLog> eventLogByBaseline = getDosEventLogByBaseline(value, destinationIp, attackType);
Tuple2<Severity, DosEventLog> eventLogByStaticThreshold = getDosEventLogByStaticThreshold(value, thresholdMap);
finalResult = mergeFinalResult(eventLogByBaseline, eventLogByStaticThreshold);
} else {
logger.debug("未获取到当前server IP：{} 类型 {} 静态阈值 和 baseline", destinationIp, attackType);
}
} catch (Exception e) {
logger.error("判定失败\n {} \n{}", value, e);
}
return finalResult;
}
private DosEventLog mergeFinalResult(DosEventLog eventLogByBaseline,DosEventLog eventLogByStaticThreshold){
return eventLogByStaticThreshold;
private DosEventLog mergeFinalResult(Tuple2<Severity, DosEventLog> eventLogByBaseline, Tuple2<Severity, DosEventLog> eventLogByStaticThreshold) {
if (eventLogByBaseline.f0.score > eventLogByStaticThreshold.f0.score) {
mergeCondition(eventLogByBaseline.f1, eventLogByStaticThreshold.f1);
logger.info("merge eventLogByBaseline {} \neventLogByStaticThreshold {}",eventLogByBaseline,eventLogByStaticThreshold);
return eventLogByBaseline.f1;
} else {
mergeCondition(eventLogByStaticThreshold.f1, eventLogByBaseline.f1);
logger.info("merge eventLogByStaticThreshold {} \neventLogByBaseline {}",eventLogByStaticThreshold,eventLogByBaseline);
return eventLogByStaticThreshold.f1;
}
}
private DosEventLog getDosEventLogByBaseline(DosSketchLog value, String destinationIp, String attackType) throws ParseException {
private void mergeCondition(DosEventLog log1, DosEventLog log2) {
if (log1 != null && log2 != null) {
String conditions1 = log1.getConditions();
String conditions2 = log2.getConditions();
log1.setConditions(conditions1 + " and " + conditions2);
}
}
private Tuple2<Severity, DosEventLog> getDosEventLogByBaseline(DosSketchLog value, String destinationIp, String attackType) throws ParseException {
Tuple2<ArrayList<Integer>, Integer> floodTypeTup = baselineMap.get(destinationIp).get(attackType);
Integer base = getBaseValue(floodTypeTup, value);
long diff = value.getSketch_sessions() - base;
return getDosEventLog(value, base, diff);
return getDosEventLog(value, base, diff, "baseline");
}
private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, Map<String, DosDetectionThreshold> thresholdMap) throws ParseException {
DosEventLog result = null;
private Tuple2<Severity, DosEventLog> getDosEventLogByStaticThreshold(DosSketchLog value, Map<String, DosDetectionThreshold> thresholdMap) throws ParseException {
Tuple2<Severity, DosEventLog> result = Tuple2.of(Severity.NORMAL, null);
String attackType = value.getAttack_type();
if (thresholdMap.containsKey(attackType)) {
DosDetectionThreshold threshold = thresholdMap.get(attackType);
long base = threshold.getSessionsPerSec();
long diff = value.getSketch_sessions() - base;
result = getDosEventLog(value, base, diff);
result = getDosEventLog(value, base, diff, "static");
}
return result;
}
private DosEventLog getDosEventLog(DosSketchLog value, long base, long diff) throws ParseException {
private Tuple2<Severity, DosEventLog> getDosEventLog(DosSketchLog value, long base, long diff, String tag) throws ParseException {
DosEventLog result = null;
String destinationIp = value.getDestination_ip();
String attackType = value.getAttack_type();
Severity severity = Severity.NORMAL;
if (diff > 0 && base != 0) {
String percent = getDiffPercent(diff, base);
double diffPercentDouble = getDiffPercentDouble(percent);
Severity severity = judgeSeverity(diffPercentDouble);
severity = judgeSeverity(diffPercentDouble);
if (severity != Severity.NORMAL) {
result = getResult(value, severity, percent);
result = getResult(value, severity, percent, tag);
logger.info("检测到当前server IP {} 存在 {} 异常,日志详情\n {}", destinationIp, attackType, result.toString());
} else {
logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value.toString());
}
}
return result;
return Tuple2.of(severity, result);
}
private DosEventLog getResult(DosSketchLog value, Severity severity, String percent) {
private DosEventLog getResult(DosSketchLog value, Severity severity, String percent, String tag) {
DosEventLog dosEventLog = new DosEventLog();
dosEventLog.setLog_id(SnowflakeId.generateId());
dosEventLog.setStart_time(value.getSketch_start_time());
dosEventLog.setEnd_time(value.getSketch_start_time() + CommonConfig.FLINK_WINDOW_MAX_TIME);
dosEventLog.setAttack_type(value.getAttack_type());
dosEventLog.setSeverity(severity.toString());
dosEventLog.setConditions(getConditions(percent));
dosEventLog.setSeverity(severity.severity);
dosEventLog.setConditions(getConditions(percent, value.getSketch_sessions(), tag));
dosEventLog.setDestination_ip(value.getDestination_ip());
dosEventLog.setDestination_country(IpUtils.ipLookup.countryLookup(value.getDestination_ip()));
String ipList = value.getSource_ip();
@@ -146,8 +181,15 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return base;
}
private String getConditions(String percent) {
return "sessions > " + percent + " of baseline";
private String getConditions(String percent, long sessions, String tag) {
switch (tag) {
case "baseline":
return "sessions > " + percent + " of baseline";
case "static":
return "sessions > " + sessions + " sessions/s";
default:
return null;
}
}
private String getSourceCountryList(String sourceIpList) {
@@ -195,22 +237,24 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
/**
* 判断严重程度枚举类型
*/
CRITICAL("Critical"),
SEVERE("Severe"),
MAJOR("Major"),
WARNING("Warning"),
MINOR("Minor"),
NORMAL("Normal");
CRITICAL("Critical", 5),
SEVERE("Severe", 4),
MAJOR("Major", 3),
WARNING("Warning", 2),
MINOR("Minor", 1),
NORMAL("Normal", 0);
private final String severity;
private final int score;
@Override
public String toString() {
return this.severity;
}
Severity(String severity) {
Severity(String severity, int score) {
this.severity = severity;
this.score = score;
}
}