gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加基于packets与bits作为static条件判断依据。

Browse Source 
修复static配置IP冲突问题。
This commit is contained in:
wanglihui
2021-10-19 18:39:13 +08:00
parent b03ab9642d
commit c692112445
3 changed files with 104 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
src/main/java/com/zdjizhi/etl/DosDetection.java
View File

@@ -4,13 +4,11 @@ import com.zdjizhi.common.CommonConfig;
import com.zdjizhi.common.DosDetectionThreshold;
import com.zdjizhi.common.DosEventLog;
import com.zdjizhi.common.DosSketchLog;
import com.zdjizhi.utils.DateUtils;
import com.zdjizhi.utils.HbaseUtils;
import com.zdjizhi.utils.IpUtils;
import com.zdjizhi.utils.SnowflakeId;
import com.zdjizhi.utils.*;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressString;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.text.StrBuilder;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -82,7 +80,7 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
DosEventLog result = null;
long sketchSessions = value.getSketch_sessions();
if (sketchSessions > CommonConfig.STATIC_SENSITIVITY_THRESHOLD) {
result = getDosEventLog(value, CommonConfig.STATIC_SENSITIVITY_THRESHOLD, sketchSessions - CommonConfig.STATIC_SENSITIVITY_THRESHOLD, "sensitivity");
result = getDosEventLog(value, CommonConfig.STATIC_SENSITIVITY_THRESHOLD, sketchSessions - CommonConfig.STATIC_SENSITIVITY_THRESHOLD, 3, "sessions");
result.setSeverity(Severity.MAJOR.severity);
}
return result;
@@ -96,7 +94,7 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
if (sketchSessions > CommonConfig.STATIC_SENSITIVITY_THRESHOLD) {
Tuple2<ArrayList<Integer>, Integer> floodTypeTup = baselineMap.get(destinationIp).get(attackType);
Integer base = getBaseValue(floodTypeTup, value);
result = getDosEventLog(value, base, sketchSessions - base, "baseline");
result = getDosEventLog(value, base, sketchSessions - base, 2, "sessions");
}
return result;
}
@@ -104,16 +102,27 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, Map<String, DosDetectionThreshold> thresholdMap) {
DosEventLog result = null;
String attackType = value.getAttack_type();
long base, diff;
if (thresholdMap.containsKey(attackType)) {
DosDetectionThreshold threshold = thresholdMap.get(attackType);
long base = threshold.getSessionsPerSec();
long diff = value.getSketch_sessions() - base;
result = getDosEventLog(value, base, diff, "static");
base = threshold.getSessionsPerSec();
diff = value.getSketch_sessions() - base;
result = getDosEventLog(value, base, diff, 1, "sessions");
if (result == null) {
base = threshold.getPacketsPerSec();
diff = value.getSketch_packets() - base;
result = getDosEventLog(value, base, diff, 1, "packets");
if (result == null) {
base = threshold.getBitsPerSec();
diff = value.getSketch_bytes() - base;
result = getDosEventLog(value, base, diff, 1, "bits");
}
}
}
return result;
}
private DosEventLog getDosEventLog(DosSketchLog value, long base, long diff, String tag) {
private DosEventLog getDosEventLog(DosSketchLog value, long base, long diff, int type, String tag) {
DosEventLog result = null;
String destinationIp = value.getDestination_ip();
String attackType = value.getAttack_type();
@@ -121,11 +130,11 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
double percent = getDiffPercent(diff, base);
Severity severity = judgeSeverity(percent);
if (severity != Severity.NORMAL) {
if ("baseline".equals(tag) && percent < CommonConfig.BASELINE_SENSITIVITY_THRESHOLD){
if (type == 2 && percent < CommonConfig.BASELINE_SENSITIVITY_THRESHOLD) {
logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过基线敏感阈值,日志详情\n{}", destinationIp, attackType, base, percent, value);
} else {
result = getResult(value,base, severity, percent, tag);
logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,基于{}检测,日志详情\n {}", destinationIp,attackType,base,percent,tag,result);
result = getResult(value, base, severity, percent, type, tag);
logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,基于{}检测,日志详情\n {}", destinationIp, attackType, base, percent, type, result);
}
} else {
logger.debug("当前server IP:{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value);
@@ -134,14 +143,14 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return result;
}
private DosEventLog getResult(DosSketchLog value,long base, Severity severity, double percent, String tag) {
private DosEventLog getResult(DosSketchLog value, long base, Severity severity, double percent, int type, String tag) {
DosEventLog dosEventLog = new DosEventLog();
dosEventLog.setLog_id(SnowflakeId.generateId());
dosEventLog.setStart_time(value.getSketch_start_time());
dosEventLog.setEnd_time(value.getSketch_start_time() + value.getSketch_duration());
dosEventLog.setAttack_type(value.getAttack_type());
dosEventLog.setSeverity(severity.severity);
dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent),base, value.getSketch_sessions(), tag));
dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent), base, value.getSketch_sessions(), type, tag));
dosEventLog.setDestination_ip(value.getDestination_ip());
dosEventLog.setDestination_country(IpUtils.ipLookup.countryLookup(value.getDestination_ip()));
String ipList = value.getSource_ip();
@@ -174,32 +183,60 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return base;
}
private String getConditions(String percent,long base, long sessions, String tag) {
switch (tag) {
case "baseline":
return "sessions > " + percent + " of baseline";
case "static":
return "sessions > " + base + " sessions/s";
case "sensitivity":
return sessions+" sessions/s Unusually high Sessions";
private String getConditions(String percent, long base, long sessions, int type, String tag) {
switch (type) {
case 1:
return new StrBuilder()
.append(tag).append(" > ")
.append(base).append(" ")
.append(tag).append("/s")
.toString();
case 2:
return new StrBuilder()
.append(tag).append(" > ")
.append(percent).append(" of baseline")
.toString();
case 3:
return new StrBuilder()
.append(sessions).append(" ")
.append(tag).append("/s Unusually high ")
.append(StringUtils.capitalize(tag))
.toString();
default:
return null;
throw new IllegalArgumentException("Illegal Argument " + type + ", known types = [1,2,3]");
}
}
private String getSourceCountryList(String sourceIpList) {
if (StringUtil.isNotBlank(sourceIpList)) {
String countryList;
try {
String[] ipArr = sourceIpList.split(",");
HashSet<String> countrySet = new HashSet<>();
for (String ip : ipArr) {
countrySet.add(IpUtils.ipLookup.countryLookup(ip));
}
return StringUtils.join(countrySet, ",");
countryList = StringUtils.join(countrySet, ",");
return countryList;
} catch (Exception e) {
logger.error("{} source IP lists 获取国家失败", sourceIpList, e);
return StringUtil.EMPTY;
}
} else {
throw new IllegalArgumentException("Illegal Argument sourceIpList = null");
}
}
private int getCurrentTimeIndex(long sketchStartTime) {
int index = 0;
try {
long currentDayTime = DateUtils.getTimeFloor(new Date(sketchStartTime * 1000L), "P1D").getTime() / 1000;
long indexLong = (sketchStartTime - currentDayTime) / 600;
return Integer.parseInt(Long.toString(indexLong));
long indexLong = (sketchStartTime - currentDayTime) / (86400 / BASELINE_SIZE);
index = Integer.parseInt(Long.toString(indexLong));
} catch (Exception e) {
logger.error("获取time index失败", e);
}
return index;
}
public static void main(String[] args) {
@@ -207,10 +244,9 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
System.out.println(date);
Date p1D = DateUtils.getTimeFloor(date, "P1D");
System.out.println(p1D + " " + p1D.getTime() / 1000);
System.out.println(new DosDetection().getCurrentTimeIndex(1631548860));
System.out.println(new DosDetection().getCurrentTimeIndex(1634659080));
System.out.println(new DosDetection().getConditions(PERCENT_INSTANCE.format(1.64862), 100, 100, 3, "packets"));
System.out.println(10 + 10 * 0.2);
Map<String, DosDetectionThreshold> thresholdMap = null;
System.out.println(thresholdMap.containsKey("a"));
}
private Double getDiffPercent(long diff, long base) {

10
src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java
View File

@@ -163,28 +163,28 @@ public class ParseStaticThreshold {
}
Map.Entry<Range<IPAddress>, Map<String, DosDetectionThreshold>> lowerEntry = thresholdRangeMap.getEntry(lower);
Map.Entry<Range<IPAddress>, Map<String, DosDetectionThreshold>> upperEntry = thresholdRangeMap.getEntry(upper);
if (lowerEntry == null && upperEntry == null){
thresholdRangeMap.put(Range.closed(lower, upper), floodTypeThresholdMap);
}else if (lowerEntry != null && upperEntry == null){
if (lowerEntry != null && upperEntry == null) {
Range<IPAddress> lowerEntryKey = lowerEntry.getKey();
Map<String, DosDetectionThreshold> lowerEntryValue = lowerEntry.getValue();
lowerEntryValue.put(threshold.getAttackType(), threshold);
thresholdRangeMap.put(Range.closedOpen(lowerEntryKey.lowerEndpoint(), lower), lowerEntryValue);
thresholdRangeMap.put(Range.closed(lower, upper), floodTypeThresholdMap);
}else if (lowerEntry == null){
} else if (lowerEntry == null && upperEntry != null) {
Range<IPAddress> upperEntryKey = upperEntry.getKey();
Map<String, DosDetectionThreshold> upperEntryValue = upperEntry.getValue();
upperEntryValue.put(threshold.getAttackType(), threshold);
thresholdRangeMap.put(Range.openClosed(upper, upperEntryKey.upperEndpoint()), upperEntryValue);
thresholdRangeMap.put(Range.closed(lower, upper), floodTypeThresholdMap);
} else {
thresholdRangeMap.put(Range.closed(lower, upper), floodTypeThresholdMap);
}
} else {
Map.Entry<Range<IPAddress>, Map<String, DosDetectionThreshold>> entry = thresholdRangeMap.getEntry(address);
if (entry != null) {
Range<IPAddress> entryKey = entry.getKey();
Map<String, DosDetectionThreshold> entryValue = entry.getValue();
entryValue.put(threshold.getAttackType(), threshold);
if (entryKey.lowerEndpoint() == entryKey.upperEndpoint()) {
entryValue.put(threshold.getAttackType(), threshold);
thresholdRangeMap.put(Range.closed(address, address), entryValue);
} else {
thresholdRangeMap.put(Range.closed(address, address), floodTypeThresholdMap);

5
src/main/resources/common.properties
View File

@@ -60,7 +60,7 @@ flink.detection.map.parallelism=1
flink.watermark.max.orderness=10
#计算窗口大小，默认600s
flink.window.max.time=600
flink.window.max.time=60
#dos event结果中distinct source IP限制
source.ip.list.limit=10000
@@ -89,7 +89,8 @@ baseline.sessions.severe.threshold=3
baseline.sessions.critical.threshold=8
#bifang服务访问地址
bifang.server.uri=http://192.168.44.72:80
#bifang.server.uri=http://192.168.44.72:80
bifang.server.uri=http://192.168.44.3:80
#访问bifang只读权限token，bifang内置，无需修改
bifang.server.token=ed04b942-7df4-4e3d-b9a9-a881ca98a867