gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复因double精度问题导致日志判定结果等级错误bug

Browse Source 
新增定时器，定时获取baseline
This commit is contained in:
wanglihui
2021-08-26 18:26:37 +08:00
parent b2936abe1a
commit b95ca44c01
4 changed files with 41 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/zdjizhi/common/CommonConfig.java
View File

@@ -45,4 +45,6 @@ public class CommonConfig {
public static final double BASELINE_SESSIONS_SEVERE_THRESHOLD = CommonConfigurations.getDoubleProperty("baseline.sessions.severe.threshold");
public static final double BASELINE_SESSIONS_CRITICAL_THRESHOLD = CommonConfigurations.getDoubleProperty("baseline.sessions.critical.threshold");
public static final int BASELINE_THRESHOLD_SCHEDULE_DAYS = CommonConfigurations.getIntProperty("baseline.threshold.schedule.days");
}

52
src/main/java/com/zdjizhi/etl/DosDetection.java
View File

@@ -7,15 +7,19 @@ import com.zdjizhi.utils.HbaseUtils;
import com.zdjizhi.utils.IpUtils;
import com.zdjizhi.utils.SnowflakeId;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.text.NumberFormat;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* @author wlh
@@ -29,7 +33,17 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
@Override
public void open(Configuration parameters) {
baselineMap = HbaseUtils.baselineMap;
ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(2,
new BasicThreadFactory.Builder().namingPattern("Dos-Detection-%d").daemon(true).build());
try {
executorService.scheduleAtFixedRate(() -> {
//do something
baselineMap = HbaseUtils.readFromHbase();
}, 0, CommonConfig.BASELINE_THRESHOLD_SCHEDULE_DAYS, TimeUnit.DAYS);
}catch (Exception e){
logger.error("定时器任务执行失败", e);
}
PERCENT_INSTANCE.setMinimumFractionDigits(2);
}
@@ -44,15 +58,14 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
Integer base = getBaseValue(floodTypeTup, value);
long diff = value.getSketch_sessions() - base;
if (diff > 0 && base != 0) {
String percent = getDiffPercent(diff, base);
double diffPercentDouble = getDiffPercentDouble(percent);
Severity severity = judgeSeverity(diffPercentDouble);
double percent = getDiffPercent(diff, base);
Severity severity = judgeSeverity(percent);
if (severity != Severity.NORMAL) {
DosEventLog result = getResult(value, severity, percent);
logger.info("检测到当前server IP {} 存在 {} 异常,日志详情\n {}", destinationIp, attackType, result.toString());
logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,日志详情\n {}", destinationIp, attackType, base, percent, result);
return result;
} else {
logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value.toString());
logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value);
}
}
} else {
@@ -64,14 +77,14 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return null;
}
private DosEventLog getResult(DosSketchLog value, Severity severity, String percent) {
private DosEventLog getResult(DosSketchLog value, Severity severity, double percent) {
DosEventLog dosEventLog = new DosEventLog();
dosEventLog.setLog_id(SnowflakeId.generateId());
dosEventLog.setStart_time(value.getSketch_start_time());
dosEventLog.setEnd_time(value.getSketch_start_time() + CommonConfig.FLINK_WINDOW_MAX_TIME);
dosEventLog.setAttack_type(value.getAttack_type());
dosEventLog.setSeverity(severity.toString());
dosEventLog.setConditions(getConditions(percent));
dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent)));
dosEventLog.setDestination_ip(value.getDestination_ip());
dosEventLog.setDestination_country(IpUtils.ipLookup.countryLookup(value.getDestination_ip()));
String ipList = value.getSource_ip();
@@ -86,7 +99,7 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
private Integer getBaseValue(Tuple2<ArrayList<Integer>, Integer> floodTypeTup, DosSketchLog value) {
Integer base = 0;
try {
if (floodTypeTup != null){
if (floodTypeTup != null) {
ArrayList<Integer> baselines = floodTypeTup.f0;
Integer defaultVaule = floodTypeTup.f1;
if (baselines != null && baselines.size() == BASELINE_SIZE) {
@@ -123,19 +136,16 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return Integer.parseInt(Long.toString(indexLong));
}
private String getDiffPercent(long diff, long base) {
double diffDou = Double.parseDouble(Long.toString(diff));
double baseDou = Double.parseDouble(Long.toString(base));
return PERCENT_INSTANCE.format(diffDou / baseDou);
private Double getDiffPercent(long diff, long base) {
return BigDecimal.valueOf((float) diff / base).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue();
}
public static void main(String[] args) throws Exception {
System.out.println(new DosDetection().getDiffPercent(219, 0));
System.out.println(new DosDetection().getDiffPercentDouble("∞%"));
}
private double getDiffPercentDouble(String diffPercent) throws ParseException {
return PERCENT_INSTANCE.parse(diffPercent).doubleValue();
public static void main(String[] args) {
DosDetection dosDetection = new DosDetection();
double diffPercent = dosDetection.getDiffPercent(135, 17);
System.out.println(diffPercent);
System.out.println(dosDetection.judgeSeverity(4.2857142857142856E14));
System.out.println(BigDecimal.valueOf((float) 10 / 3).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue());
}
private Severity judgeSeverity(double diffPercent) {

7
src/main/java/com/zdjizhi/utils/HbaseUtils.java
View File

@@ -25,7 +25,6 @@ public class HbaseUtils {
private static final Logger logger = LoggerFactory.getLogger(HbaseUtils.class);
private static Table table = null;
private static Scan scan = null;
public static Map<String, Map<String, Tuple2<ArrayList<Integer>, Integer>>> baselineMap = new HashMap<>();
private static ArrayList<String> floodTypeList = new ArrayList<>();
static {
@@ -33,7 +32,6 @@ public class HbaseUtils {
floodTypeList.add("UDP Flood");
floodTypeList.add("ICMP Flood");
floodTypeList.add("DNS Amplification");
readFromHbase();
}
private static void prepareHbaseEnv() throws IOException {
@@ -54,6 +52,7 @@ public class HbaseUtils {
}
public static void main(String[] args) {
Map<String, Map<String, Tuple2<ArrayList<Integer>, Integer>>> baselineMap = readFromHbase();
Set<String> keySet = baselineMap.keySet();
for (String key : keySet) {
Map<String, Tuple2<ArrayList<Integer>, Integer>> stringTuple2Map = baselineMap.get(key);
@@ -66,7 +65,8 @@ public class HbaseUtils {
System.out.println(baselineMap.size());
}
private static void readFromHbase() {
public static Map<String, Map<String, Tuple2<ArrayList<Integer>, Integer>>> readFromHbase() {
Map<String, Map<String, Tuple2<ArrayList<Integer>, Integer>>> baselineMap = new HashMap<>();
try {
prepareHbaseEnv();
logger.info("开始读取baseline数据");
@@ -87,6 +87,7 @@ public class HbaseUtils {
} catch (Exception e) {
logger.error("读取hbase数据失败", e);
}
return baselineMap;
}
private static Integer getDefaultValue(Result result, String family, String qualifier) {