gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-15219 修复静态阈值的condition处理逻辑，新增静态阈值单元测试类

Browse Source
This commit is contained in:
unknown
2023-05-26 15:44:37 +08:00
parent 6be3ea7f1e
commit 72acc976e3
3 changed files with 271 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/main/java/com/zdjizhi/etl/DosDetection.java
View File

@@ -121,22 +121,30 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
}
private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, DosDetectionThreshold threshold) throws CloneNotSupportedException {
long base = threshold.getSessionsPerSec();
long diffSession = value.getSketch_sessions() - base;
long diffPkt = value.getSketch_packets() - base;
long diffByte = value.getSketch_bytes() - base;
long sessionBase = threshold.getSessionsPerSec();
long pktBase=threshold.getPacketsPerSec();
long bitBase=threshold.getBitsPerSec();
long diffSession = value.getSketch_sessions() - sessionBase;
long diffPkt = value.getSketch_packets() - pktBase;
long diffByte = value.getSketch_bytes() - bitBase;
Double diffSessionPercent = getDiffPercent(diffSession, sessionBase)*100;
Double diffPktPercent = getDiffPercent(diffPkt, pktBase)*100;
Double diffBitPercent = getDiffPercent(diffByte, bitBase)*100;
long profileId = 0;
DosEventLog result =null;
// long max =(value.getSketch_sessions()>value.getSketch_packets()?value.getSketch_sessions():value.getSketch_packets())>value.getSketch_bytes()?(value.getSketch_sessions()>value.getSketch_packets()?value.getSketch_sessions():value.getSketch_packets()):value.getSketch_bytes();
if (diffSession>diffPkt && diffSession> diffByte){
if (diffSessionPercent >= diffPktPercent && diffSessionPercent >= diffBitPercent){
profileId = threshold.getProfileId();
result= getDosEventLog(value, base, diffSession, profileId, STATIC_CONDITION_TYPE, SESSIONS_TAG);
}else if (diffPkt>diffSession && diffPkt>diffByte){
result= getDosEventLog(value, sessionBase, diffSession, profileId, STATIC_CONDITION_TYPE, SESSIONS_TAG);
}else if (diffPktPercent >= diffSessionPercent && diffPktPercent >= diffBitPercent){
profileId = threshold.getProfileId();
result = getDosEventLog(value, base, diffPkt,profileId, STATIC_CONDITION_TYPE, PACKETS_TAG);
}else if (diffByte>diffPkt&&diffByte>diffSession){
result = getDosEventLog(value, pktBase, diffPkt,profileId, STATIC_CONDITION_TYPE, PACKETS_TAG);
}else if (diffBitPercent >= diffPktPercent && diffBitPercent >= diffSessionPercent){
profileId = threshold.getProfileId();
result = getDosEventLog(value, base, diffByte, profileId, STATIC_CONDITION_TYPE, BITS_TAG);
result = getDosEventLog(value, bitBase, diffByte, profileId, STATIC_CONDITION_TYPE, BITS_TAG);
}
@@ -210,8 +218,8 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
dosEventLog.setProfile_id(profileId);
dosEventLog.setAttack_type(value.getAttack_type());
dosEventLog.setSeverity(severity.severity);
// dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent), base, value.getSketch_sessions(), type, tag));
dosEventLog.setConditions(getConditions(percent, base, value.getSketch_sessions(), type, tag,dosEventLog));
dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent), base, value.getSketch_sessions(), type, tag,dosEventLog));
// dosEventLog.setConditions(getConditions(percent, base, value.getSketch_sessions(), type, tag,dosEventLog));
dosEventLog.setDestination_ip(value.getDestination_ip());
dosEventLog.setDestination_country(IpUtils.ipLookup.countryLookup(value.getDestination_ip()));
String ipList = value.getSource_ip();
@@ -248,7 +256,7 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
return base;
}
private String getConditions(double percent, long base, long sessions, int type, String tag,DosEventLog dosEventLog) {
private String getConditions(String percent, long base, long sessions, int type, String tag,DosEventLog dosEventLog) {
int condition =0;
if ("Minor".equals(dosEventLog.getSeverity())){
condition=50;
@@ -268,7 +276,7 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
tag + "/s" + "(>"+condition+"%)";
case BASELINE_CONDITION_TYPE:
return tag + " > " +
PERCENT_INSTANCE.format(percent) + " of baseline";
percent + " of baseline";
case SENSITIVITY_CONDITION_TYPE:
return String.valueOf(sessions) + " " +
tag + "/s Unusually high " +

237
src/test/java/com/zdjizhi/etl/DosDetectionTest.java Normal file
View File

@@ -0,0 +1,237 @@
package com.zdjizhi.etl;
import com.zdjizhi.common.DosDetectionThreshold;
import com.zdjizhi.common.DosEventLog;
import com.zdjizhi.common.DosSketchLog;
import com.zdjizhi.utils.IpUtils;
import com.zdjizhi.utils.NacosUtils;
import com.zdjizhi.utils.SnowflakeId;
import com.zdjizhi.utils.StringUtil;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;
import java.math.BigDecimal;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.HashSet;
public class DosDetectionTest {
private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance();
private final static int BASELINE_SIZE = 144;
private final static int STATIC_CONDITION_TYPE = 1;
private final static int BASELINE_CONDITION_TYPE = 2;
private final static int SENSITIVITY_CONDITION_TYPE = 3;
private final static String SESSIONS_TAG = "sessions";
private final static String PACKETS_TAG = "packets";
private final static String BITS_TAG = "bits";
@Test
public void dosDetectionTest(){
DosDetectionThreshold dosDetectionThreshold = new DosDetectionThreshold();
ArrayList<String> serverIpList = new ArrayList<>();
serverIpList.add("192.168.50.11");
serverIpList.add("192.168.50.1/24");
serverIpList.add("FC::12:0:0/54");
serverIpList.add("FC::12:0:0");
dosDetectionThreshold.setProfileId(4437);
dosDetectionThreshold.setAttackType("DNS Flood");
dosDetectionThreshold.setServerIpList(serverIpList);
dosDetectionThreshold.setSessionsPerSec(1);
dosDetectionThreshold.setPacketsPerSec(1);
dosDetectionThreshold.setBitsPerSec(100000);
dosDetectionThreshold.setIsValid(1);
dosDetectionThreshold.setSuperiorIds(new Integer[]{5,4,12,27});
DosSketchLog dosSketchLog = new DosSketchLog ();
dosSketchLog.setSketch_sessions(68);
dosSketchLog.setSketch_packets(68);
dosSketchLog.setSketch_bytes(285820);//185.82
dosSketchLog.setVsys_id(1);
dosSketchLog.setAttack_type("ICMP Flood");
dosSketchLog.setSource_ip("45.170.244.25");
dosSketchLog.setDestination_ip("24.152.57.56");
//静态阈值获取
long sessionBase = dosDetectionThreshold.getSessionsPerSec();
long pktBase=dosDetectionThreshold.getPacketsPerSec();
long bitBase=dosDetectionThreshold.getBitsPerSec();
//基于速率进行计算
long diffSession = dosSketchLog.getSketch_sessions() - sessionBase;
long diffPkt = dosSketchLog.getSketch_packets() - pktBase;
long diffByte = dosSketchLog.getSketch_bytes() - bitBase;
Double diffSessionPercent = getDiffPercent(diffSession, sessionBase)*100;
Double diffPktPercent = getDiffPercent(diffPkt, pktBase)*100;
Double diffBitPercent = getDiffPercent(diffByte, bitBase)*100;
long profileId = 0;
DosEventLog result =null;
if (diffSessionPercent >= diffPktPercent && diffSessionPercent >= diffBitPercent){
profileId = dosDetectionThreshold.getProfileId();
result= getDosEventLog(dosSketchLog, sessionBase, diffSession, profileId, STATIC_CONDITION_TYPE, SESSIONS_TAG);
System.out.println(result);
}else if (diffPktPercent >= diffSessionPercent && diffPktPercent >= diffBitPercent){
profileId = dosDetectionThreshold.getProfileId();
result = getDosEventLog(dosSketchLog, pktBase, diffPkt,profileId, STATIC_CONDITION_TYPE, PACKETS_TAG);
System.out.println(result);
}else if (diffBitPercent >= diffPktPercent && diffBitPercent >= diffSessionPercent){
profileId = dosDetectionThreshold.getProfileId();
result = getDosEventLog(dosSketchLog, bitBase, diffByte, profileId, STATIC_CONDITION_TYPE, BITS_TAG);
System.out.println(result);
}
}
private DosEventLog getDosEventLog(DosSketchLog value, long base, long diff, long profileId, int type, String tag) {
DosEventLog result = null;
String destinationIp = value.getDestination_ip();
String attackType = value.getAttack_type();
if (diff > 0 && base != 0) {
double percent = getDiffPercent(diff, base);
Severity severity = judgeSeverity(percent);
Integer staticSensitivityThreshold = 100;
if (severity != Severity.NORMAL) {
if (type == BASELINE_CONDITION_TYPE && percent < 0.2) {
// logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过基线敏感阈值,日志详情\n{}", destinationIp, attackType, base, percent, value);
}else if ((type == BASELINE_CONDITION_TYPE || type == SENSITIVITY_CONDITION_TYPE) && value.getSketch_sessions() < staticSensitivityThreshold){
// logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过静态敏感阈值,日志详情\n{}",destinationIp, attackType, base, percent, value);
}else {
result = getResult(value, base, profileId, severity, percent+1, type, tag);
if (type == SENSITIVITY_CONDITION_TYPE){
result.setSeverity(Severity.MAJOR.severity);
}
// logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,基于{}:{}检测,日志详情\n {}", destinationIp,attackType,base,percent,type,tag,result);
}
}
// else {
// logger.debug("当前server IP:{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value);
// }
}
return result;
}
private DosEventLog getResult(DosSketchLog value, long base, long profileId, Severity severity, double percent, int type, String tag) {
DosEventLog dosEventLog = new DosEventLog();
// dosEventLog.setLog_id(SnowflakeId.generateId());
dosEventLog.setVsys_id(value.getVsys_id());
dosEventLog.setStart_time(value.getSketch_start_time());
dosEventLog.setEnd_time(value.getSketch_start_time() + value.getSketch_duration());
dosEventLog.setProfile_id(profileId);
dosEventLog.setAttack_type(value.getAttack_type());
dosEventLog.setSeverity(severity.severity);
// dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent), base, value.getSketch_sessions(), type, tag));
dosEventLog.setConditions(getConditions(percent, base, value.getSketch_sessions(), type, tag,dosEventLog));
dosEventLog.setDestination_ip(value.getDestination_ip());
// dosEventLog.setDestination_country(IpUtils.ipLookup.countryLookup(value.getDestination_ip()));
String ipList = value.getSource_ip();
dosEventLog.setSource_ip_list(ipList);
dosEventLog.setSource_country_list(getSourceCountryList(ipList));
dosEventLog.setSession_rate(value.getSketch_sessions());
dosEventLog.setPacket_rate(value.getSketch_packets());
dosEventLog.setBit_rate(value.getSketch_bytes());
return dosEventLog;
}
public String getConditions(double percent, long base, long sessions, int type, String tag,DosEventLog dosEventLog) {
int condition =0;
if ("Minor".equals(dosEventLog.getSeverity())){
condition=50;
}else if ("Warning".equals(dosEventLog.getSeverity())){
condition=100;
}else if ("Major".equals(dosEventLog.getSeverity())){
condition=250;
}else if ("Severe".equals(dosEventLog.getSeverity())){
condition=500;
}else if ("Critical".equals(dosEventLog.getSeverity())){
condition =800;
}
switch (type) {
case STATIC_CONDITION_TYPE:
return "Rate > " +
base + " " +
tag + "/s" + "(>"+condition+"%)";
case BASELINE_CONDITION_TYPE:
return tag + " > " +
PERCENT_INSTANCE.format(percent) + " of baseline";
case SENSITIVITY_CONDITION_TYPE:
return String.valueOf(sessions) + " " +
tag + "/s Unusually high " +
StringUtils.capitalize(tag);
default:
throw new IllegalArgumentException("Illegal Argument type:" + type + ", known types = [1,2,3]");
}
}
private String getSourceCountryList(String sourceIpList) {
if (StringUtil.isNotBlank(sourceIpList)) {
String countryList;
try {
String[] ipArr = sourceIpList.split(",");
HashSet<String> countrySet = new HashSet<>();
for (String ip : ipArr) {
String country = IpUtils.ipLookup.countryLookup(ip);
if (StringUtil.isNotBlank(country)){
countrySet.add(country);
}
}
countryList = StringUtils.join(countrySet, ", ");
return countryList;
} catch (Exception e) {
// logger.error("{} source IP lists 获取国家失败", sourceIpList, e);
return StringUtil.EMPTY;
}
} else {
throw new IllegalArgumentException("Illegal Argument sourceIpList = null");
}
}
private Double getDiffPercent(long diff, long base) {
return BigDecimal.valueOf((float) diff / base).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue();
}
private Severity judgeSeverity(double diffPercent) {
if (diffPercent >= 0.5 && diffPercent < 1) {
return Severity.MINOR;
} else if (diffPercent >= 1 && diffPercent < 2.5) {
return Severity.WARNING;
} else if (diffPercent >= 2.5 && diffPercent < 5) {
return Severity.MAJOR;
} else if (diffPercent >= 5 && diffPercent < 8) {
return Severity.SEVERE;
} else if (diffPercent >= 8) {
return Severity.CRITICAL;
} else {
return Severity.NORMAL;
}
}
private enum Severity {
/**
* 判断严重程度枚举类型
*/
CRITICAL("Critical"),
SEVERE("Severe"),
MAJOR("Major"),
WARNING("Warning"),
MINOR("Minor"),
NORMAL("Normal");
private final String severity;
@Override
public String toString() {
return this.severity;
}
Severity(String severity) {
this.severity = severity;
}
}
}

11
src/test/java/com/zdjizhi/etl/EtlProcessFunctionTest.java Normal file
View File

@@ -0,0 +1,11 @@
package com.zdjizhi.etl;
import org.junit.Test;
public class EtlProcessFunctionTest {
@Test
public void EtlProcessFunctionTest(){
}
}