diff --git a/src/main/java/com/zdjizhi/common/CommonConfig.java b/src/main/java/com/zdjizhi/common/CommonConfig.java index 2ff146d..aa64f95 100644 --- a/src/main/java/com/zdjizhi/common/CommonConfig.java +++ b/src/main/java/com/zdjizhi/common/CommonConfig.java @@ -45,4 +45,6 @@ public class CommonConfig { public static final double BASELINE_SESSIONS_SEVERE_THRESHOLD = CommonConfigurations.getDoubleProperty("baseline.sessions.severe.threshold"); public static final double BASELINE_SESSIONS_CRITICAL_THRESHOLD = CommonConfigurations.getDoubleProperty("baseline.sessions.critical.threshold"); + public static final int BASELINE_THRESHOLD_SCHEDULE_DAYS = CommonConfigurations.getIntProperty("baseline.threshold.schedule.days"); + } diff --git a/src/main/java/com/zdjizhi/etl/DosDetection.java b/src/main/java/com/zdjizhi/etl/DosDetection.java index d6d82ea..22dc76d 100644 --- a/src/main/java/com/zdjizhi/etl/DosDetection.java +++ b/src/main/java/com/zdjizhi/etl/DosDetection.java @@ -7,15 +7,19 @@ import com.zdjizhi.utils.HbaseUtils; import com.zdjizhi.utils.IpUtils; import com.zdjizhi.utils.SnowflakeId; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.math.BigDecimal; import java.text.NumberFormat; -import java.text.ParseException; import java.util.*; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * @author wlh @@ -29,7 +33,17 @@ public class DosDetection extends RichMapFunction { @Override public void open(Configuration parameters) { - baselineMap = HbaseUtils.baselineMap; + ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(2, + new BasicThreadFactory.Builder().namingPattern("Dos-Detection-%d").daemon(true).build()); + try { + executorService.scheduleAtFixedRate(() -> { + //do something + baselineMap = HbaseUtils.readFromHbase(); + }, 0, CommonConfig.BASELINE_THRESHOLD_SCHEDULE_DAYS, TimeUnit.DAYS); + + }catch (Exception e){ + logger.error("定时器任务执行失败", e); + } PERCENT_INSTANCE.setMinimumFractionDigits(2); } @@ -44,15 +58,14 @@ public class DosDetection extends RichMapFunction { Integer base = getBaseValue(floodTypeTup, value); long diff = value.getSketch_sessions() - base; if (diff > 0 && base != 0) { - String percent = getDiffPercent(diff, base); - double diffPercentDouble = getDiffPercentDouble(percent); - Severity severity = judgeSeverity(diffPercentDouble); + double percent = getDiffPercent(diff, base); + Severity severity = judgeSeverity(percent); if (severity != Severity.NORMAL) { DosEventLog result = getResult(value, severity, percent); - logger.info("检测到当前server IP {} 存在 {} 异常,日志详情\n {}", destinationIp, attackType, result.toString()); + logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,日志详情\n {}", destinationIp, attackType, base, percent, result); return result; } else { - logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value.toString()); + logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value); } } } else { @@ -64,14 +77,14 @@ public class DosDetection extends RichMapFunction { return null; } - private DosEventLog getResult(DosSketchLog value, Severity severity, String percent) { + private DosEventLog getResult(DosSketchLog value, Severity severity, double percent) { DosEventLog dosEventLog = new DosEventLog(); dosEventLog.setLog_id(SnowflakeId.generateId()); dosEventLog.setStart_time(value.getSketch_start_time()); dosEventLog.setEnd_time(value.getSketch_start_time() + CommonConfig.FLINK_WINDOW_MAX_TIME); dosEventLog.setAttack_type(value.getAttack_type()); dosEventLog.setSeverity(severity.toString()); - dosEventLog.setConditions(getConditions(percent)); + dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent))); dosEventLog.setDestination_ip(value.getDestination_ip()); dosEventLog.setDestination_country(IpUtils.ipLookup.countryLookup(value.getDestination_ip())); String ipList = value.getSource_ip(); @@ -86,7 +99,7 @@ public class DosDetection extends RichMapFunction { private Integer getBaseValue(Tuple2, Integer> floodTypeTup, DosSketchLog value) { Integer base = 0; try { - if (floodTypeTup != null){ + if (floodTypeTup != null) { ArrayList baselines = floodTypeTup.f0; Integer defaultVaule = floodTypeTup.f1; if (baselines != null && baselines.size() == BASELINE_SIZE) { @@ -123,19 +136,16 @@ public class DosDetection extends RichMapFunction { return Integer.parseInt(Long.toString(indexLong)); } - private String getDiffPercent(long diff, long base) { - double diffDou = Double.parseDouble(Long.toString(diff)); - double baseDou = Double.parseDouble(Long.toString(base)); - return PERCENT_INSTANCE.format(diffDou / baseDou); + private Double getDiffPercent(long diff, long base) { + return BigDecimal.valueOf((float) diff / base).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue(); } - public static void main(String[] args) throws Exception { - System.out.println(new DosDetection().getDiffPercent(219, 0)); - System.out.println(new DosDetection().getDiffPercentDouble("∞%")); - } - - private double getDiffPercentDouble(String diffPercent) throws ParseException { - return PERCENT_INSTANCE.parse(diffPercent).doubleValue(); + public static void main(String[] args) { + DosDetection dosDetection = new DosDetection(); + double diffPercent = dosDetection.getDiffPercent(135, 17); + System.out.println(diffPercent); + System.out.println(dosDetection.judgeSeverity(4.2857142857142856E14)); + System.out.println(BigDecimal.valueOf((float) 10 / 3).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue()); } private Severity judgeSeverity(double diffPercent) { diff --git a/src/main/java/com/zdjizhi/utils/HbaseUtils.java b/src/main/java/com/zdjizhi/utils/HbaseUtils.java index bf9ced5..c147e61 100644 --- a/src/main/java/com/zdjizhi/utils/HbaseUtils.java +++ b/src/main/java/com/zdjizhi/utils/HbaseUtils.java @@ -25,7 +25,6 @@ public class HbaseUtils { private static final Logger logger = LoggerFactory.getLogger(HbaseUtils.class); private static Table table = null; private static Scan scan = null; - public static Map, Integer>>> baselineMap = new HashMap<>(); private static ArrayList floodTypeList = new ArrayList<>(); static { @@ -33,7 +32,6 @@ public class HbaseUtils { floodTypeList.add("UDP Flood"); floodTypeList.add("ICMP Flood"); floodTypeList.add("DNS Amplification"); - readFromHbase(); } private static void prepareHbaseEnv() throws IOException { @@ -54,6 +52,7 @@ public class HbaseUtils { } public static void main(String[] args) { + Map, Integer>>> baselineMap = readFromHbase(); Set keySet = baselineMap.keySet(); for (String key : keySet) { Map, Integer>> stringTuple2Map = baselineMap.get(key); @@ -66,7 +65,8 @@ public class HbaseUtils { System.out.println(baselineMap.size()); } - private static void readFromHbase() { + public static Map, Integer>>> readFromHbase() { + Map, Integer>>> baselineMap = new HashMap<>(); try { prepareHbaseEnv(); logger.info("开始读取baseline数据"); @@ -87,6 +87,7 @@ public class HbaseUtils { } catch (Exception e) { logger.error("读取hbase数据失败", e); } + return baselineMap; } private static Integer getDefaultValue(Result result, String family, String qualifier) { diff --git a/src/main/resources/common.properties b/src/main/resources/common.properties index 3700f90..bee457b 100644 --- a/src/main/resources/common.properties +++ b/src/main/resources/common.properties @@ -80,4 +80,7 @@ baseline.sessions.minor.threshold=0.1 baseline.sessions.warning.threshold=0.5 baseline.sessions.major.threshold=1 baseline.sessions.severe.threshold=3 -baseline.sessions.critical.threshold=8 \ No newline at end of file +baseline.sessions.critical.threshold=8 + +#获取baseline周期，默认7天 +baseline.threshold.schedule.days=7 \ No newline at end of file