diff --git a/src/main/java/com/zdjizhi/etl/DosDetection.java b/src/main/java/com/zdjizhi/etl/DosDetection.java index d6d82ea..f572107 100644 --- a/src/main/java/com/zdjizhi/etl/DosDetection.java +++ b/src/main/java/com/zdjizhi/etl/DosDetection.java @@ -1,15 +1,19 @@ package com.zdjizhi.etl; import com.zdjizhi.common.CommonConfig; +import com.zdjizhi.common.DosDetectionThreshold; import com.zdjizhi.common.DosEventLog; import com.zdjizhi.common.DosSketchLog; import com.zdjizhi.utils.HbaseUtils; import com.zdjizhi.utils.IpUtils; import com.zdjizhi.utils.SnowflakeId; +import inet.ipaddr.IPAddress; +import inet.ipaddr.IPAddressString; import org.apache.commons.lang.StringUtils; import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; +import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,42 +30,80 @@ public class DosDetection extends RichMapFunction { private static Map, Integer>>> baselineMap; private final static int BASELINE_SIZE = 144; private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance(); + private TreeRangeMap> thresholdRangeMap; @Override public void open(Configuration parameters) { baselineMap = HbaseUtils.baselineMap; + thresholdRangeMap = ParseStaticThreshold.createStaticThreshold(); PERCENT_INSTANCE.setMinimumFractionDigits(2); } @Override public DosEventLog map(DosSketchLog value) { + DosEventLog finalResult = null; try { String destinationIp = value.getDestination_ip(); String attackType = value.getAttack_type(); + IPAddress destinationIpAddress = new IPAddressString(destinationIp).getAddress(); + Map thresholdMap = thresholdRangeMap.get(destinationIpAddress); logger.debug("当前判断IP：{}, 类型: {}", destinationIp, attackType); - if (baselineMap.containsKey(destinationIp)) { - Tuple2, Integer> floodTypeTup = baselineMap.get(destinationIp).get(attackType); - Integer base = getBaseValue(floodTypeTup, value); - long diff = value.getSketch_sessions() - base; - if (diff > 0 && base != 0) { - String percent = getDiffPercent(diff, base); - double diffPercentDouble = getDiffPercentDouble(percent); - Severity severity = judgeSeverity(diffPercentDouble); - if (severity != Severity.NORMAL) { - DosEventLog result = getResult(value, severity, percent); - logger.info("检测到当前server IP {} 存在 {} 异常,日志详情\n {}", destinationIp, attackType, result.toString()); - return result; - } else { - logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value.toString()); - } - } - } else { - logger.debug("未获取到当前server IP：{} 类型 {} baseline数据", destinationIp, attackType); + if (baselineMap.containsKey(destinationIp) && thresholdMap == null) { + finalResult = getDosEventLogByBaseline(value, destinationIp, attackType); + } else if (!baselineMap.containsKey(destinationIp) && thresholdMap != null) { + finalResult = getDosEventLogByStaticThreshold(value,thresholdMap); + }else if (baselineMap.containsKey(destinationIp) && thresholdMap != null){ + DosEventLog eventLogByBaseline = getDosEventLogByBaseline(value, destinationIp, attackType); + DosEventLog eventLogByStaticThreshold = getDosEventLogByStaticThreshold(value, thresholdMap); + finalResult = mergeFinalResult(eventLogByBaseline,eventLogByStaticThreshold); + }else { + logger.debug("未获取到当前server IP：{} 类型 {} 静态阈值 和 baseline", destinationIp, attackType); } } catch (Exception e) { logger.error("判定失败\n {} \n{}", value, e); } - return null; + return finalResult; + } + + private DosEventLog mergeFinalResult(DosEventLog eventLogByBaseline,DosEventLog eventLogByStaticThreshold){ + return eventLogByStaticThreshold; + } + + private DosEventLog getDosEventLogByBaseline(DosSketchLog value, String destinationIp, String attackType) throws ParseException { + Tuple2, Integer> floodTypeTup = baselineMap.get(destinationIp).get(attackType); + Integer base = getBaseValue(floodTypeTup, value); + long diff = value.getSketch_sessions() - base; + return getDosEventLog(value, base, diff); + } + + private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, Map thresholdMap) throws ParseException { + DosEventLog result = null; + String attackType = value.getAttack_type(); + if (thresholdMap.containsKey(attackType)) { + DosDetectionThreshold threshold = thresholdMap.get(attackType); + long base = threshold.getSessionsPerSec(); + long diff = value.getSketch_sessions() - base; + result = getDosEventLog(value, base, diff); + } + return result; + } + + private DosEventLog getDosEventLog(DosSketchLog value, long base, long diff) throws ParseException { + DosEventLog result = null; + String destinationIp = value.getDestination_ip(); + String attackType = value.getAttack_type(); + if (diff > 0 && base != 0) { + String percent = getDiffPercent(diff, base); + double diffPercentDouble = getDiffPercentDouble(percent); + Severity severity = judgeSeverity(diffPercentDouble); + if (severity != Severity.NORMAL) { + result = getResult(value, severity, percent); + logger.info("检测到当前server IP {} 存在 {} 异常,日志详情\n {}", destinationIp, attackType, result.toString()); + } else { + logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value.toString()); + } + } + return result; } private DosEventLog getResult(DosSketchLog value, Severity severity, String percent) { @@ -86,7 +128,7 @@ public class DosDetection extends RichMapFunction { private Integer getBaseValue(Tuple2, Integer> floodTypeTup, DosSketchLog value) { Integer base = 0; try { - if (floodTypeTup != null){ + if (floodTypeTup != null) { ArrayList baselines = floodTypeTup.f0; Integer defaultVaule = floodTypeTup.f1; if (baselines != null && baselines.size() == BASELINE_SIZE) { @@ -129,11 +171,6 @@ public class DosDetection extends RichMapFunction { return PERCENT_INSTANCE.format(diffDou / baseDou); } - public static void main(String[] args) throws Exception { - System.out.println(new DosDetection().getDiffPercent(219, 0)); - System.out.println(new DosDetection().getDiffPercentDouble("∞%")); - } - private double getDiffPercentDouble(String diffPercent) throws ParseException { return PERCENT_INSTANCE.parse(diffPercent).doubleValue(); } diff --git a/src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java b/src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java index 40559ed..1a246f7 100644 --- a/src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java +++ b/src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java @@ -125,8 +125,8 @@ public class ParseStaticThreshold { * 基于静态阈值构建threshold RangeMap，k:IP段或具体IP，v:配置信息 * @return threshold RangeMap */ - public static TreeRangeMap createStaticThreshold(){ - TreeRangeMap thresholdRangeMap = null; + public static TreeRangeMap> createStaticThreshold(){ + TreeRangeMap> thresholdRangeMap = null; try { ArrayList dosDetectionThreshold = getDosDetectionThreshold(); if (dosDetectionThreshold != null && !dosDetectionThreshold.isEmpty()){ @@ -137,7 +137,12 @@ public class ParseStaticThreshold { IPAddressString ipAddressString = new IPAddressString(sip); if (ipAddressString.isIPAddress()){ IPAddress address = ipAddressString.getAddress(); - thresholdRangeMap.put(Range.closed(address.getLower(),address.getUpper()),threshold); + Map floodTypeThresholdMap = thresholdRangeMap.get(address); + if (floodTypeThresholdMap == null){ + floodTypeThresholdMap = new HashMap<>(); + } + floodTypeThresholdMap.put(threshold.getAttackType(),threshold); + thresholdRangeMap.put(Range.closed(address.getLower(),address.getUpper()),floodTypeThresholdMap); } } } @@ -149,14 +154,15 @@ public class ParseStaticThreshold { } public static void main(String[] args) { - - TreeRangeMap staticThreshold = createStaticThreshold(); - Map, DosDetectionThreshold> rangeDosDetectionThresholdMap = staticThreshold.asMapOfRanges(); - Set> ranges = rangeDosDetectionThresholdMap.keySet(); - for (Range range:ranges){ - System.out.println(range+"--"+rangeDosDetectionThresholdMap.get(range)); + TreeRangeMap> staticThreshold = createStaticThreshold(); + Map, Map> rangeMapMap = staticThreshold.asMapOfRanges(); + for (Range range:rangeMapMap.keySet()){ + Map thresholdMap = rangeMapMap.get(range); + for (String type:thresholdMap.keySet()){ + DosDetectionThreshold threshold = thresholdMap.get(type); + System.out.println(range+"---"+type+"---"+threshold); + } } - } diff --git a/src/main/java/com/zdjizhi/utils/IpUtils.java b/src/main/java/com/zdjizhi/utils/IpUtils.java index d8576c8..4f7e5d7 100644 --- a/src/main/java/com/zdjizhi/utils/IpUtils.java +++ b/src/main/java/com/zdjizhi/utils/IpUtils.java @@ -9,9 +9,9 @@ public class IpUtils { */ public static IpLookup ipLookup = new IpLookup.Builder(false) .loadDataFileV4(CommonConfig.IP_MMDB_PATH + "ip_v4.mmdb") -// .loadDataFileV6(CommonConfig.IP_MMDB_PATH + "ip_v6.mmdb") -// .loadDataFilePrivateV4(CommonConfig.IP_MMDB_PATH + "ip_private_v4.mmdb") -// .loadDataFilePrivateV6(CommonConfig.IP_MMDB_PATH + "ip_private_v6.mmdb") + .loadDataFileV6(CommonConfig.IP_MMDB_PATH + "ip_v6.mmdb") + .loadDataFilePrivateV4(CommonConfig.IP_MMDB_PATH + "ip_private_v4.mmdb") + .loadDataFilePrivateV6(CommonConfig.IP_MMDB_PATH + "ip_private_v6.mmdb") .build(); public static void main(String[] args) { diff --git a/src/test/java/com/zdjizhi/common/IpTest.java b/src/test/java/com/zdjizhi/common/IpTest.java index 830f7bf..b522634 100644 --- a/src/test/java/com/zdjizhi/common/IpTest.java +++ b/src/test/java/com/zdjizhi/common/IpTest.java @@ -13,6 +13,7 @@ import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap; import java.util.Arrays; import java.util.HashMap; +import java.util.Map; public class IpTest { public static void main(String[] args) throws Exception { @@ -40,6 +41,11 @@ public class IpTest { IPAddress pv43 = new IPAddressString("fc00::").getAddress(); IPAddress pv44 = new IPAddressString("fc00::10:1").getAddress(); + IPAddress pv45 = new IPAddressString("12.56.4.0/32").getAddress(); + System.out.println(str5.getUpper()+"---"+str5.getLower()); + + System.out.println(rangeMap.span().contains(pv4)); + System.out.println(rangeMap.get(pv4)); System.out.println(rangeMap.get(pv42)); System.out.println(rangeMap.get(pv43));