gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

基于DoS Sketch一元组进行实时检测

Browse Source
This commit is contained in:
wanglihui
2021-09-14 18:46:23 +08:00
parent 8cfb442c44
commit 62f3c65d66
9 changed files with 118 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
src/main/java/com/zdjizhi/etl/DosDetection.java
View File

@@ -65,15 +65,13 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
IPAddress destinationIpAddress = new IPAddressString(destinationIp).getAddress();
Map<String, DosDetectionThreshold> thresholdMap = thresholdRangeMap.get(destinationIpAddress);
logger.debug("当前判断IP：{}, 类型: {}", destinationIp, attackType);
if (baselineMap != null && baselineMap.containsKey(destinationIp) && thresholdMap == null) {
finalResult = getDosEventLogByBaseline(value, destinationIp, attackType).f1;
} else if (baselineMap != null && !baselineMap.containsKey(destinationIp) && thresholdMap != null) {
finalResult = getDosEventLogByStaticThreshold(value, thresholdMap).f1;
} else if (baselineMap != null && baselineMap.containsKey(destinationIp) && thresholdMap != null) {
Tuple2<Severity, DosEventLog> eventLogByBaseline = getDosEventLogByBaseline(value, destinationIp, attackType);
Tuple2<Severity, DosEventLog> eventLogByStaticThreshold = getDosEventLogByStaticThreshold(value, thresholdMap);
finalResult = mergeFinalResult(eventLogByBaseline, eventLogByStaticThreshold);
} else {
if (thresholdMap == null && baselineMap.containsKey(destinationIp)) {
finalResult = getDosEventLogByBaseline(value, destinationIp, attackType);
}else if (thresholdMap == null && !baselineMap.containsKey(destinationIp)){
finalResult = getDosEventLogBySensitivityThreshold(value);
} else if (thresholdMap != null){
finalResult = getDosEventLogByStaticThreshold(value, thresholdMap);
}else {
logger.debug("未获取到当前server IP：{} 类型 {} 静态阈值 和 baseline", destinationIp, attackType);
}
@@ -83,37 +81,29 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return finalResult;
}
private DosEventLog mergeFinalResult(Tuple2<Severity, DosEventLog> eventLogByBaseline, Tuple2<Severity, DosEventLog> eventLogByStaticThreshold) {
if (eventLogByBaseline.f0.score > eventLogByStaticThreshold.f0.score) {
logger.info("merge eventLogByBaseline {} \neventLogByStaticThreshold {}",eventLogByBaseline,eventLogByStaticThreshold);
return mergeCondition(eventLogByBaseline.f1, eventLogByStaticThreshold.f1);
} else {
logger.info("merge eventLogByStaticThreshold {} \neventLogByBaseline {}",eventLogByStaticThreshold,eventLogByBaseline);
return mergeCondition(eventLogByStaticThreshold.f1, eventLogByBaseline.f1);
}
}
private DosEventLog mergeCondition(DosEventLog log1, DosEventLog log2) {
if (log1 != null && log2 != null) {
String conditions1 = log1.getConditions();
String conditions2 = log2.getConditions();
log1.setConditions(conditions1 + " and " + conditions2);
}else if (log1 == null && log2 != null){
log1 = log2;
}
return log1;
}
private Tuple2<Severity, DosEventLog> getDosEventLogByBaseline(DosSketchLog value, String destinationIp, String attackType) {
Tuple2<ArrayList<Integer>, Integer> floodTypeTup = baselineMap.get(destinationIp).get(attackType);
Integer base = getBaseValue(floodTypeTup, value);
private DosEventLog getDosEventLogBySensitivityThreshold(DosSketchLog value){
DosEventLog result = null;
long sketchSessions = value.getSketch_sessions();
return sketchSessions > CommonConfig.SENSITIVITY_THRESHOLD ?
getDosEventLog(value, base, sketchSessions - base, "baseline") : Tuple2.of(Severity.NORMAL, null);
if (sketchSessions > CommonConfig.SENSITIVITY_THRESHOLD){
result = getDosEventLog(value, CommonConfig.SENSITIVITY_THRESHOLD, sketchSessions - CommonConfig.SENSITIVITY_THRESHOLD, "sensitivity");
result.setSeverity(Severity.MAJOR.severity);
}
return result;
}
private Tuple2<Severity, DosEventLog> getDosEventLogByStaticThreshold(DosSketchLog value, Map<String, DosDetectionThreshold> thresholdMap) {
Tuple2<Severity, DosEventLog> result = Tuple2.of(Severity.NORMAL, null);
private DosEventLog getDosEventLogByBaseline(DosSketchLog value, String destinationIp, String attackType) {
DosEventLog result = null;
long sketchSessions = value.getSketch_sessions();
if (sketchSessions > CommonConfig.SENSITIVITY_THRESHOLD){
Tuple2<ArrayList<Integer>, Integer> floodTypeTup = baselineMap.get(destinationIp).get(attackType);
Integer base = getBaseValue(floodTypeTup, value);
result = getDosEventLog(value, base, sketchSessions - base, "baseline");
}
return result;
}
private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, Map<String, DosDetectionThreshold> thresholdMap) {
DosEventLog result = null;
String attackType = value.getAttack_type();
if (thresholdMap.containsKey(attackType)) {
DosDetectionThreshold threshold = thresholdMap.get(attackType);
@@ -124,14 +114,13 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return result;
}
private Tuple2<Severity, DosEventLog> getDosEventLog(DosSketchLog value, long base, long diff, String tag) {
private DosEventLog getDosEventLog(DosSketchLog value, long base, long diff, String tag) {
DosEventLog result = null;
String destinationIp = value.getDestination_ip();
String attackType = value.getAttack_type();
Severity severity = Severity.NORMAL;
if (diff > 0 && base != 0) {
double percent = getDiffPercent(diff, base);
severity = judgeSeverity(percent);
Severity severity = judgeSeverity(percent);
if (severity != Severity.NORMAL) {
result = getResult(value, severity, percent, tag);
logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,日志详情\n {}", destinationIp,attackType,base,percent,result);
@@ -139,7 +128,7 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value.toString());
}
}
return Tuple2.of(severity, result);
return result;
}
private DosEventLog getResult(DosSketchLog value, Severity severity, double percent, String tag) {
@@ -188,6 +177,8 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return "sessions > " + percent + " of baseline";
case "static":
return "sessions > " + sessions + " sessions/s";
case "sensitivity":
return sessions+" sessions/s Unusually high Sessions";
default:
return null;
}
@@ -208,6 +199,11 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return Integer.parseInt(Long.toString(indexLong));
}
public static void main(String[] args) {
System.out.println(1631579940 / (60 * 60) * 60 * 60);
System.out.println(new DosDetection().getCurrentTimeIndex(1631579940));
}
private Double getDiffPercent(long diff, long base) {
return BigDecimal.valueOf((float)diff/base).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue();
}
@@ -254,4 +250,27 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
}
@Deprecated
private DosEventLog mergeFinalResult(Tuple2<Severity, DosEventLog> eventLogByBaseline, Tuple2<Severity, DosEventLog> eventLogByStaticThreshold) {
if (eventLogByBaseline.f0.score > eventLogByStaticThreshold.f0.score) {
logger.info("merge eventLogByBaseline {} \neventLogByStaticThreshold {}",eventLogByBaseline,eventLogByStaticThreshold);
return mergeCondition(eventLogByBaseline.f1, eventLogByStaticThreshold.f1);
} else {
logger.info("merge eventLogByStaticThreshold {} \neventLogByBaseline {}",eventLogByStaticThreshold,eventLogByBaseline);
return mergeCondition(eventLogByStaticThreshold.f1, eventLogByBaseline.f1);
}
}
@Deprecated
private DosEventLog mergeCondition(DosEventLog log1, DosEventLog log2) {
if (log1 != null && log2 != null) {
String conditions1 = log1.getConditions();
String conditions2 = log2.getConditions();
log1.setConditions(conditions1 + " and " + conditions2);
}else if (log1 == null && log2 != null){
log1 = log2;
}
return log1;
}
}