gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-20112，TSG-20099适配功能端日志调整，重构代码

Browse Source
This commit is contained in:
wangkuan
2024-04-03 17:29:22 +08:00
parent fae16e5804
commit 49f1200787
22 changed files with 600 additions and 1522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
src/main/java/com/zdjizhi/function/DosDetectionFunction.java
View File

@@ -3,18 +3,12 @@ package com.zdjizhi.function;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.geedgenetworks.utils.DateUtils;
import com.geedgenetworks.utils.StringUtil;
import com.zdjizhi.common.*;
import com.zdjizhi.utils.Snowflakeld.SnowflakeId;
import com.zdjizhi.utils.Threshold.ParseBaselineThreshold;
import com.zdjizhi.utils.Threshold.ParseStaticThreshold;
import com.zdjizhi.utils.connections.http.HttpClientService;
import com.zdjizhi.utils.knowledgebase.IpLookupUtils;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressString;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
@@ -31,7 +25,7 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
private static final Log logger = LogFactory.get();
private Map<String, Map<String, DosBaselineThreshold>> baselineMap = new HashMap<>();
private final NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance();
private HashMap<Integer, HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> thresholdRangeMap;
// private HashMap<Integer, HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> thresholdRangeMap;
private final int BASELINE_SIZE = 144;
private final int STATIC_CONDITION_TYPE = 1;
private final int BASELINE_CONDITION_TYPE = 2;
@@ -42,51 +36,17 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
private final int OTHER_BASELINE_TYPE = 3;
private SnowflakeId snowflakeId;
private Configuration configuration;
private HttpClientService httpClientService;
private IpLookupUtils ipLookupUtils;
//private IpLookupUtils ipLookupUtils;
private ParseBaselineThreshold parseBaselineThresholdld;
private ParseStaticThreshold parseStaticThreshold;
@Override
public void open(Configuration parameters) {
configuration = (Configuration) getRuntimeContext()
.getExecutionConfig().getGlobalJobParameters();
httpClientService = new HttpClientService(configuration);
snowflakeId = new SnowflakeId(configuration.get(DATA_CENTER_ID_NUM), getRuntimeContext().getIndexOfThisSubtask());
try {
ipLookupUtils = new IpLookupUtils(configuration, httpClientService);
ipLookupUtils.stuffKnowledgeMetaCache();
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
ipLookupUtils.stuffKnowledgeMetaCache();
logger.info("定位库定时调度成功");
}
}, configuration.get(KNOWLEDGE_BASE_SCHEDULE_MINUTES) * 60 * 1000, configuration.get(KNOWLEDGE_BASE_SCHEDULE_MINUTES) * 60 * 1000);
} catch (Exception e) {
logger.error("定位库加载失败,具体原因为" + e);
}
try {
parseStaticThreshold = new ParseStaticThreshold(configuration, httpClientService);
thresholdRangeMap = parseStaticThreshold.createStaticThreshold();
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
thresholdRangeMap = parseStaticThreshold.createStaticThreshold();
logger.info("基于静态阈值构建threshold RangeMap成功,Threshold RangeMap:" + thresholdRangeMap.toString());
}
}, configuration.get(STATIC_THRESHOLD_SCHEDULE_MINUTES) * 60 * 1000, configuration.get(STATIC_THRESHOLD_SCHEDULE_MINUTES) * 60 * 1000);
} catch (Exception e) {
logger.error("基于静态阈值构建threshold RangeMap失败,失败原因为:" + e);
}
try {
parseBaselineThresholdld = new ParseBaselineThreshold(configuration);
baselineMap = parseBaselineThresholdld.readFromHbase();
@@ -95,7 +55,7 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
@Override
public void run() {
baselineMap = parseBaselineThresholdld.readFromHbase();
logger.info("从Hbase获取baselineMap成功,baselineMap:" + thresholdRangeMap.toString());
logger.info("从Hbase获取baselineMap成功,baselineMap:" + baselineMap.toString());
}
}, configuration.get(BASELINE_THRESHOLD_SCHEDULE_DAYS) * 24 * 60 * 60 * 1000, configuration.get(BASELINE_THRESHOLD_SCHEDULE_DAYS) * 24 * 60 * 60 * 1000);
@@ -110,31 +70,29 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
public void processElement(DosSketchLog value, Context ctx, Collector<DosEventLog> out) throws Exception {
DosEventLog finalResult = null;
try {
String destinationIp = value.getDestination_ip();
int vsysId = value.getVsys_id();
String key = destinationIp + "-" + vsysId;
String attackType = value.getAttack_type();
IPAddress destinationIpAddress = new IPAddressString(destinationIp).getAddress();
DosDetectionThreshold threshold = null;
if (thresholdRangeMap.containsKey(vsysId)) {
threshold = thresholdRangeMap.get(vsysId).getOrDefault(attackType, TreeRangeMap.create()).get(destinationIpAddress);
if (value.getRule_id() == 0) {
String destinationIp = value.getServer_ip();
int vsysId = value.getVsys_id();
String key = destinationIp + "-" + vsysId;
String attackType = value.getAttack_type();
DosDetectionThreshold threshold = null;
logger.debug("当前判断IP：{}, 类型: {}", key, attackType);
if (threshold == null && baselineMap.containsKey(key)) {
finalResult = getDosEventLogByBaseline(value, key);
} else if (threshold == null && !baselineMap.containsKey(key)) {
finalResult = getDosEventLogBySensitivityThreshold(value);
}
else {
logger.debug("未获取到当前server IP：{} 类型 {} 静态阈值 和 baseline", key, attackType);
}
}
else{
finalResult = getResult(value,0,0,Severity.MAJOR,0.0,0,"DoS Protection [12]");
}
} catch(Exception e){
logger.error("判定失败\n {} \n{}", value, e);
}
logger.debug("当前判断IP：{}, 类型: {}", key, attackType);
if (threshold == null && baselineMap.containsKey(key)) {
finalResult = getDosEventLogByBaseline(value, key);
} else if (threshold == null && !baselineMap.containsKey(key)) {
finalResult = getDosEventLogBySensitivityThreshold(value);
} else if (threshold != null) {
finalResult = getDosEventLogByStaticThreshold(value, threshold);
} else {
logger.debug("未获取到当前server IP：{} 类型 {} 静态阈值 和 baseline", key, attackType);
}
} catch (Exception e) {
logger.error("判定失败\n {} \n{}", value, e);
}
if (finalResult != null) {
out.collect(finalResult);
}
@@ -142,7 +100,7 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
private DosEventLog getDosEventLogBySensitivityThreshold(DosSketchLog value) {
long sketchSessions = value.getSketch_sessions();
long sketchSessions = value.getSessions();
Integer staticSensitivityThreshold = configuration.get(STATIC_SENSITIVITY_THRESHOLD);
long diff = sketchSessions - staticSensitivityThreshold;
return getDosEventLog(value, staticSensitivityThreshold, diff, 0, SENSITIVITY_CONDITION_TYPE, SESSIONS_TAG);
@@ -150,55 +108,16 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
private DosEventLog getDosEventLogByBaseline(DosSketchLog value, String key) {
String attackType = value.getAttack_type();
long sketchSessions = value.getSketch_sessions();
long sketchSessions = value.getSessions();
DosBaselineThreshold dosBaselineThreshold = baselineMap.get(key).get(attackType);
Integer base = getBaseValue(dosBaselineThreshold, value);
long diff = sketchSessions - base;
return getDosEventLog(value, base, diff, 0, BASELINE_CONDITION_TYPE, SESSIONS_TAG);
}
private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, DosDetectionThreshold threshold) throws CloneNotSupportedException {
long sessionBase = threshold.getSessions_per_sec();
long pktBase = threshold.getPackets_per_sec();
long bitBase = threshold.getBits_per_sec();
long diffSession = value.getSketch_sessions() - sessionBase;
long diffPkt = value.getSketch_packets() - pktBase;
long diffByte = value.getSketch_bytes() - bitBase;
double diffSessionPercent = 0.0;
double diffPktPercent = 0.0;
double diffBitPercent = 0.0;
if (sessionBase > 0) {
diffSessionPercent = getDiffPercent(diffSession, sessionBase) * 100;
}
if (pktBase > 0) {
diffPktPercent = getDiffPercent(diffPkt, pktBase) * 100;
}
if (bitBase > 0) {
diffBitPercent = getDiffPercent(diffByte, bitBase) * 100;
}
long profileId = 0;
DosEventLog result = null;
if (diffSessionPercent >= diffPktPercent && diffSessionPercent >= diffBitPercent) {
profileId = threshold.getId();
result = getDosEventLog(value, sessionBase, diffSession, profileId, STATIC_CONDITION_TYPE, SESSIONS_TAG);
} else if (diffPktPercent >= diffSessionPercent && diffPktPercent >= diffBitPercent) {
profileId = threshold.getId();
result = getDosEventLog(value, pktBase, diffPkt, profileId, STATIC_CONDITION_TYPE, PACKETS_TAG);
} else if (diffBitPercent >= diffPktPercent && diffBitPercent >= diffSessionPercent) {
profileId = threshold.getId();
result = getDosEventLog(value, bitBase, diffByte, profileId, STATIC_CONDITION_TYPE, BITS_TAG);
}
return result;
}
private DosEventLog getDosEventLog(DosSketchLog value, long base, long diff, long profileId, int type, String tag) {
DosEventLog result = null;
String destinationIp = value.getDestination_ip();
String destinationIp = value.getServer_ip();
String attackType = value.getAttack_type();
if (diff > 0 && base != 0) {
double percent = getDiffPercent(diff, base);
@@ -207,7 +126,7 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
if (severity != Severity.NORMAL) {
if (type == BASELINE_CONDITION_TYPE && percent < configuration.get(BASELINE_SENSITIVITY_THRESHOLD)) {
logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过基线敏感阈值,日志详情\n{}", destinationIp, attackType, base, percent, value);
} else if ((type == BASELINE_CONDITION_TYPE || type == SENSITIVITY_CONDITION_TYPE) && value.getSketch_sessions() < staticSensitivityThreshold) {
} else if ((type == BASELINE_CONDITION_TYPE || type == SENSITIVITY_CONDITION_TYPE) && value.getSessions() < staticSensitivityThreshold) {
logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过静态敏感阈值,日志详情\n{}", destinationIp, attackType, base, percent, value);
} else {
result = getResult(value, base, profileId, severity, percent, type, tag);
@@ -225,23 +144,46 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
private DosEventLog getResult(DosSketchLog value, long base, long profileId, Severity severity, double percent, int type, String tag) {
DosEventLog dosEventLog = new DosEventLog();
dosEventLog.setRecv_time(value.getCommon_recv_time());
dosEventLog.setRecv_time(value.getRecv_time());
dosEventLog.setLog_id(snowflakeId.nextId());
dosEventLog.setVsys_id(value.getVsys_id());
dosEventLog.setStart_time(value.getSketch_start_time());
dosEventLog.setEnd_time(value.getSketch_start_time() + value.getSketch_duration());
dosEventLog.setStart_time(value.getStart_timestamp_ms()/1000);
dosEventLog.setEnd_time(value.getEnd_timestamp_ms()/1000);
dosEventLog.setProfile_id(profileId);
dosEventLog.setRule_id(value.getRule_id());
dosEventLog.setAttack_type(value.getAttack_type());
dosEventLog.setSeverity(severity.severity);
dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent), base, value.getSketch_sessions(), type, tag, dosEventLog));
dosEventLog.setDestination_ip(value.getDestination_ip());
dosEventLog.setDestination_country(ipLookupUtils.getCountryLookup(value.getDestination_ip()));
String ipList = value.getSource_ip();
dosEventLog.setSource_ip_list(ipList);
dosEventLog.setSource_country_list(getSourceCountryList(ipList));
dosEventLog.setSession_rate(value.getSketch_sessions());
dosEventLog.setPacket_rate(value.getSketch_packets());
dosEventLog.setBit_rate(value.getSketch_bytes());
if(base != 0) {
dosEventLog.setSeverity(severity.severity);
dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent), base, value.getSessions(), type, tag, dosEventLog));
}
else{
dosEventLog.setSeverity(severity.severity);
dosEventLog.setConditions(tag);
}
dosEventLog.setDestination_ip(value.getServer_ip());
dosEventLog.setDestination_country(value.getServer_country());
StringBuilder client_ips = new StringBuilder();
StringBuilder client_countrys = new StringBuilder();
Iterator<Map.Entry<String, String>> iterator = value.getClientips_countrys().entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, String> entry = iterator.next();
client_ips.append(entry.getKey());
client_countrys.append(entry.getValue());
if (iterator.hasNext()) {
client_ips.append(",");
client_countrys.append(",");
}
}
if(client_ips.length()>0){
dosEventLog.setSource_ip_list(client_ips.toString());
dosEventLog.setSource_country_list(client_countrys.toString());
}
dosEventLog.setSession_rate(value.getSession_rate());
dosEventLog.setPacket_rate(value.getPacket_rate());
dosEventLog.setBit_rate(value.getBit_rate());
dosEventLog.setBytes(value.getBytes());
dosEventLog.setSessions(value.getSessions());
dosEventLog.setPackets(value.getPkts());
return dosEventLog;
}
@@ -253,10 +195,10 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
Integer defaultVaule = dosBaselineThreshold.getSession_rate_default_value();
Integer sessionRateBaselineType = dosBaselineThreshold.getSession_rate_baseline_type();
if (baselines != null && baselines.size() == BASELINE_SIZE) {
int timeIndex = getCurrentTimeIndex(value.getSketch_start_time());
int timeIndex = getCurrentTimeIndex(value.getStart_timestamp_ms());
base = baselines.get(timeIndex);
if (base == 0) {
logger.debug("获取到当前IP: {},类型: {} baseline值为0,替换为P95观测值{}", value.getDestination_ip(), value.getAttack_type(), defaultVaule);
logger.debug("获取到当前IP: {},类型: {} baseline值为0,替换为P95观测值{}", value.getServer_ip(), value.getAttack_type(), defaultVaule);
base = defaultVaule;
}
if (sessionRateBaselineType == OTHER_BASELINE_TYPE && base < configuration.get(STATIC_SENSITIVITY_THRESHOLD)) {
@@ -300,29 +242,6 @@ public class DosDetectionFunction extends ProcessFunction<DosSketchLog, DosEvent
}
}
private String getSourceCountryList(String sourceIpList) {
if (StringUtil.isNotBlank(sourceIpList)) {
String countryList;
try {
String[] ipArr = sourceIpList.split(",");
HashSet<String> countrySet = new HashSet<>();
for (String ip : ipArr) {
String country = ipLookupUtils.getCountryLookup(ip);
if (StringUtil.isNotBlank(country)) {
countrySet.add(country);
}
}
countryList = StringUtils.join(countrySet, ", ");
return countryList;
} catch (Exception e) {
logger.error("{} source IP lists 获取国家失败", sourceIpList, e);
return StringUtil.EMPTY;
}
} else {
throw new IllegalArgumentException("Illegal Argument sourceIpList = null");
}
}
private int getCurrentTimeIndex(long sketchStartTime) {
int index = 0;
try {