TSG-15167 LTS版本新增知识库文件校验功能

pom.xml

@@ -98,8 +98,27 @@
                     </execution>
                 </executions>
             </plugin>
-
         </plugins>
+
+        <resources>
+            <resource>
+                <directory>properties</directory>
+                <includes>
+                    <include>**/*.properties</include>
+                    <include>**/*.xml</include>
+                </includes>
+                <filtering>false</filtering>
+            </resource>
+
+            <resource>
+                <directory>src\main</directory>
+                <includes>
+                    <include>log4j.properties</include>
+                </includes>
+                <filtering>false</filtering>
+            </resource>
+        </resources>
+
     </build>
 
     <dependencies>
@@ -111,17 +130,17 @@
             <version>1.9.3</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.21</version>
-        </dependency>
+        <!--<dependency>-->
+            <!--<groupId>org.slf4j</groupId>-->
+            <!--<artifactId>slf4j-api</artifactId>-->
+            <!--<version>1.7.21</version>-->
+        <!--</dependency>-->
 
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.21</version>
-        </dependency>
+        <!--<dependency>-->
+            <!--<groupId>org.slf4j</groupId>-->
+            <!--<artifactId>slf4j-log4j12</artifactId>-->
+            <!--<version>1.7.21</version>-->
+        <!--</dependency>-->
 
         <dependency>
             <groupId>com.jayway.jsonpath</groupId>
@@ -147,6 +166,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>2.7.1</version>
+            <!--<scope>provided</scope>-->
             <exclusions>
                 <exclusion>
                     <artifactId>zookeeper</artifactId>
@@ -160,6 +180,10 @@
                     <artifactId>guava</artifactId>
                     <groupId>com.google.guava</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -185,6 +209,14 @@
                     <artifactId>log4j-over-slf4j</artifactId>
                     <groupId>org.slf4j</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>hadoop-common</artifactId>
+                    <groupId>org.apache.hadoop</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -259,6 +291,10 @@
                     <groupId>commons-codec</groupId>
                     <artifactId>commons-codec</artifactId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->

src/main/java/com/zdjizhi/common/CommonConfig.java

@@ -1,7 +1,6 @@
 package com.zdjizhi.common;
 
 import com.zdjizhi.utils.CommonConfigurations;
-import com.zdjizhi.utils.NacosUtils;
 import org.jasypt.encryption.pbe.StandardPBEStringEncryptor;
 
 /**
@@ -54,17 +53,7 @@ public class CommonConfig {
 
     public static final String IP_MMDB_PATH = CommonConfigurations.getStringProperty("ip.mmdb.path");
 
-//    public static final int STATIC_SENSITIVITY_THRESHOLD = NacosUtils.getIntProperty("static.sensitivity.threshold");
-//    public static final double BASELINE_SENSITIVITY_THRESHOLD = NacosUtils.getDoubleProperty("baseline.sensitivity.threshold");
-//
-//    public static final double BASELINE_SESSIONS_MINOR_THRESHOLD = NacosUtils.getDoubleProperty("baseline.sessions.minor.threshold");
-//    public static final double BASELINE_SESSIONS_WARNING_THRESHOLD = NacosUtils.getDoubleProperty("baseline.sessions.warning.threshold");
-//    public static final double BASELINE_SESSIONS_MAJOR_THRESHOLD = NacosUtils.getDoubleProperty("baseline.sessions.major.threshold");
-//    public static final double BASELINE_SESSIONS_SEVERE_THRESHOLD = NacosUtils.getDoubleProperty("baseline.sessions.severe.threshold");
-//    public static final double BASELINE_SESSIONS_CRITICAL_THRESHOLD = NacosUtils.getDoubleProperty("baseline.sessions.critical.threshold");
-
     public static final String BIFANG_SERVER_URI = CommonConfigurations.getStringProperty("bifang.server.uri");
-    public static final String BIFANG_SERVER_TOKEN = CommonConfigurations.getStringProperty("bifang.server.token");
     public static final String BIFANG_SERVER_ENCRYPTPWD_PATH = CommonConfigurations.getStringProperty("bifang.server.encryptpwd.path");
     public static final String BIFANG_SERVER_LOGIN_PATH = CommonConfigurations.getStringProperty("bifang.server.login.path");
     public static final String BIFANG_SERVER_POLICY_THRESHOLD_PATH = CommonConfigurations.getStringProperty("bifang.server.policy.threshold.path");
@@ -88,10 +77,12 @@ public class CommonConfig {
     public static final String NACOS_SERVER_ADDR = CommonConfigurations.getStringProperty("nacos.server.addr");
     public static final String NACOS_USERNAME = CommonConfigurations.getStringProperty("nacos.username");
     public static final String NACOS_PASSWORD = CommonConfigurations.getStringProperty("nacos.password");
+    public static final String NACOS_NAMESPACE = CommonConfigurations.getStringProperty("nacos.namespace");
     public static final String NACOS_DATA_ID = CommonConfigurations.getStringProperty("nacos.data.id");
     public static final String NACOS_GROUP = CommonConfigurations.getStringProperty("nacos.group");
     public static final int NACOS_READ_TIMEOUT = CommonConfigurations.getIntProperty("nacos.read.timeout");
 
+
     public static final String HOS_TOKEN = CommonConfigurations.getStringProperty("hos.token");
 
     public static final String CLUSTER_OR_SINGLE = CommonConfigurations.getStringProperty("cluster.or.single");

src/main/java/com/zdjizhi/common/DosDetectionThreshold.java

@@ -2,6 +2,7 @@ package com.zdjizhi.common;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Objects;
 
 /**
@@ -16,44 +17,8 @@ public class DosDetectionThreshold implements Serializable {
     private long bitsPerSec;
     private long sessionsPerSec;
     private int isValid;
-
-    @Override
-    public String toString() {
-        return "DosDetectionThreshold{" +
-                "profileId='" + profileId + '\'' +
-                ", attackType='" + attackType + '\'' +
-                ", serverIpList=" + serverIpList +
-                ", serverIpAddr='" + serverIpAddr + '\'' +
-                ", packetsPerSec=" + packetsPerSec +
-                ", bitsPerSec=" + bitsPerSec +
-                ", sessionsPerSec=" + sessionsPerSec +
-                ", isValid=" + isValid +
-                '}';
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        DosDetectionThreshold threshold = (DosDetectionThreshold) o;
-        return packetsPerSec == threshold.packetsPerSec &&
-                bitsPerSec == threshold.bitsPerSec &&
-                sessionsPerSec == threshold.sessionsPerSec &&
-                isValid == threshold.isValid &&
-                Objects.equals(profileId, threshold.profileId) &&
-                Objects.equals(attackType, threshold.attackType) &&
-                Objects.equals(serverIpList, threshold.serverIpList) &&
-                Objects.equals(serverIpAddr, threshold.serverIpAddr);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(profileId, attackType, serverIpList, serverIpAddr, packetsPerSec, bitsPerSec, sessionsPerSec, isValid);
-    }
+    private int vsysId;
+    private Integer[] superiorIds;
 
     public String getProfileId() {
         return profileId;
@@ -118,4 +83,36 @@ public class DosDetectionThreshold implements Serializable {
     public void setIsValid(int isValid) {
         this.isValid = isValid;
     }
+
+    public int getVsysId() {
+        return vsysId;
+    }
+
+    public void setVsysId(int vsysId) {
+        this.vsysId = vsysId;
+    }
+
+    public Integer[] getSuperiorIds() {
+        return superiorIds;
+    }
+
+    public void setSuperiorIds(Integer[] superiorIds) {
+        this.superiorIds = superiorIds;
+    }
+
+    @Override
+    public String toString() {
+        return "DosDetectionThreshold{" +
+                "profileId='" + profileId + '\'' +
+                ", attackType='" + attackType + '\'' +
+                ", serverIpList=" + serverIpList +
+                ", serverIpAddr='" + serverIpAddr + '\'' +
+                ", packetsPerSec=" + packetsPerSec +
+                ", bitsPerSec=" + bitsPerSec +
+                ", sessionsPerSec=" + sessionsPerSec +
+                ", isValid=" + isValid +
+                ", vsysId=" + vsysId +
+                ", superiorIds=" + Arrays.toString(superiorIds) +
+                '}';
+    }
 }

src/main/java/com/zdjizhi/common/DosEventLog.java

@@ -1,11 +1,11 @@
 package com.zdjizhi.common;
 
 import java.io.Serializable;
-import java.util.Objects;
 
-public class DosEventLog implements Serializable {
+public class DosEventLog implements Serializable,Cloneable {
 
     private long log_id;
+    private int vsys_id;
     private long start_time;
     private long end_time;
     private String attack_type;
@@ -27,6 +27,14 @@ public class DosEventLog implements Serializable {
         this.log_id = log_id;
     }
 
+    public int getVsys_id() {
+        return vsys_id;
+    }
+
+    public void setVsys_id(int vsys_id) {
+        this.vsys_id = vsys_id;
+    }
+
     public long getStart_time() {
         return start_time;
     }
@@ -125,8 +133,9 @@ public class DosEventLog implements Serializable {
 
     @Override
     public String toString() {
-        return "dosEventLog{" +
+        return "DosEventLog{" +
                 "log_id=" + log_id +
+                ", vsys_id=" + vsys_id +
                 ", start_time=" + start_time +
                 ", end_time=" + end_time +
                 ", attack_type='" + attack_type + '\'' +
@@ -143,31 +152,7 @@ public class DosEventLog implements Serializable {
     }
 
     @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof DosEventLog)) {
-            return false;
-        }
-        DosEventLog that = (DosEventLog) o;
-        return getLog_id() == that.getLog_id() &&
-                getStart_time() == that.getStart_time() &&
-                getEnd_time() == that.getEnd_time() &&
-                getSession_rate() == that.getSession_rate() &&
-                getPacket_rate() == that.getPacket_rate() &&
-                getBit_rate() == that.getBit_rate() &&
-                Objects.equals(getAttack_type(), that.getAttack_type()) &&
-                Objects.equals(getSeverity(), that.getSeverity()) &&
-                Objects.equals(getConditions(), that.getConditions()) &&
-                Objects.equals(getDestination_ip(), that.getDestination_ip()) &&
-                Objects.equals(getDestination_country(), that.getDestination_country()) &&
-                Objects.equals(getSource_ip_list(), that.getSource_ip_list()) &&
-                Objects.equals(getSource_country_list(), that.getSource_country_list());
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(getLog_id(), getStart_time(), getEnd_time(), getAttack_type(), getSeverity(), getConditions(), getDestination_ip(), getDestination_country(), getSource_ip_list(), getSource_country_list(), getSession_rate(), getPacket_rate(), getBit_rate());
+    public Object clone() throws CloneNotSupportedException {
+        return super.clone();
     }
 }

src/main/java/com/zdjizhi/common/DosMetricsLog.java

@@ -12,6 +12,7 @@ public class DosMetricsLog implements Serializable {
     private long packet_rate;
     private long bit_rate;
     private int partition_num;
+    private int vsys_id;
 
     public int getPartition_num() {
         return partition_num;
@@ -69,6 +70,14 @@ public class DosMetricsLog implements Serializable {
         this.bit_rate = bit_rate;
     }
 
+    public int getVsys_id() {
+        return vsys_id;
+    }
+
+    public void setVsys_id(int vsys_id) {
+        this.vsys_id = vsys_id;
+    }
+
     @Override
     public String toString() {
         return "DosMetricsLog{" +
@@ -79,29 +88,7 @@ public class DosMetricsLog implements Serializable {
                 ", packet_rate=" + packet_rate +
                 ", bit_rate=" + bit_rate +
                 ", partition_num=" + partition_num +
+                ", vsys_id=" + vsys_id +
                 '}';
     }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof DosMetricsLog)) {
-            return false;
-        }
-        DosMetricsLog that = (DosMetricsLog) o;
-        return getSketch_start_time() == that.getSketch_start_time() &&
-                getSession_rate() == that.getSession_rate() &&
-                getPacket_rate() == that.getPacket_rate() &&
-                getBit_rate() == that.getBit_rate() &&
-                getPartition_num() == that.getPartition_num() &&
-                Objects.equals(getAttack_type(), that.getAttack_type()) &&
-                Objects.equals(getDestination_ip(), that.getDestination_ip());
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(getSketch_start_time(), getAttack_type(), getDestination_ip(), getSession_rate(), getPacket_rate(), getBit_rate(), getPartition_num());
-    }
 }

src/main/java/com/zdjizhi/common/DosSketchLog.java

@@ -15,6 +15,7 @@ public class DosSketchLog implements Serializable {
     private long sketch_sessions;
     private long sketch_packets;
     private long sketch_bytes;
+    private int vsys_id;
 
     @Override
     public String toString() {
@@ -29,35 +30,10 @@ public class DosSketchLog implements Serializable {
                 ", sketch_sessions=" + sketch_sessions +
                 ", sketch_packets=" + sketch_packets +
                 ", sketch_bytes=" + sketch_bytes +
+                ", vsys_id=" + vsys_id +
                 '}';
     }
 
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof DosSketchLog)) {
-            return false;
-        }
-        DosSketchLog sketchLog = (DosSketchLog) o;
-        return getSketch_start_time() == sketchLog.getSketch_start_time() &&
-                getSketch_duration() == sketchLog.getSketch_duration() &&
-                getSketch_sessions() == sketchLog.getSketch_sessions() &&
-                getSketch_packets() == sketchLog.getSketch_packets() &&
-                getSketch_bytes() == sketchLog.getSketch_bytes() &&
-                Objects.equals(getCommon_sled_ip(), sketchLog.getCommon_sled_ip()) &&
-                Objects.equals(getCommon_data_center(), sketchLog.getCommon_data_center()) &&
-                Objects.equals(getAttack_type(), sketchLog.getAttack_type()) &&
-                Objects.equals(getSource_ip(), sketchLog.getSource_ip()) &&
-                Objects.equals(getDestination_ip(), sketchLog.getDestination_ip());
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(getCommon_sled_ip(), getCommon_data_center(), getSketch_start_time(), getSketch_duration(), getAttack_type(), getSource_ip(), getDestination_ip(), getSketch_sessions(), getSketch_packets(), getSketch_bytes());
-    }
-
     public String getCommon_sled_ip() {
         return common_sled_ip;
     }
@@ -137,4 +113,12 @@ public class DosSketchLog implements Serializable {
     public void setSketch_bytes(long sketch_bytes) {
         this.sketch_bytes = sketch_bytes;
     }
+
+    public int getVsys_id() {
+        return vsys_id;
+    }
+
+    public void setVsys_id(int vsys_id) {
+        this.vsys_id = vsys_id;
+    }
 }

src/main/java/com/zdjizhi/common/DosVsysId.java

@@ -1,22 +1,32 @@
 package com.zdjizhi.common;
 
-import java.util.Objects;
+import java.util.Arrays;
 
 public class DosVsysId {
-    private int vsysId;
+    private Integer id;
+    private Integer[] superiorIds;
 
-    public int getVsysId() {
-        return vsysId;
+    public Integer getId() {
+        return id;
     }
 
-    public void setVsysId(int vsysId) {
-        this.vsysId = vsysId;
+    public void setId(Integer id) {
+        this.id = id;
+    }
+
+    public Integer[] getSuperiorIds() {
+        return superiorIds;
+    }
+
+    public void setSuperiorIds(Integer[] superiorIds) {
+        this.superiorIds = superiorIds;
     }
 
     @Override
     public String toString() {
         return "DosVsysId{" +
-                "vsysId=" + vsysId +
+                "id=" + id +
+                ", superiorIds=" + Arrays.toString(superiorIds) +
                 '}';
     }
 }

src/main/java/com/zdjizhi/etl/DosDetection.java

@@ -1,17 +1,17 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.*;
 import com.zdjizhi.utils.*;
 import inet.ipaddr.IPAddress;
 import inet.ipaddr.IPAddressString;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.text.StrBuilder;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
-import org.apache.flink.api.common.functions.RichMapFunction;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction;
+import org.apache.flink.util.Collector;
 
 import java.math.BigDecimal;
 import java.text.NumberFormat;
@@ -23,12 +23,12 @@ import java.util.concurrent.TimeUnit;
 /**
  * @author wlh
  */
-public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
+public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<String, byte[]>, DosEventLog> {
 
-    private static final Logger logger = LoggerFactory.getLogger(DosDetection.class);
+    private static final Log logger = LogFactory.get();
     private static Map<String, Map<String, DosBaselineThreshold>> baselineMap = new HashMap<>();
     private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance();
-    private HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>> thresholdRangeMap;
+    private HashMap<Integer,HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> thresholdRangeMap;
 
     private final static int BASELINE_SIZE = 144;
     private final static int STATIC_CONDITION_TYPE = 1;
@@ -46,6 +46,12 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
         ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(2,
                 new BasicThreadFactory.Builder().namingPattern("Dos-Detection-%d").daemon(true).build());
         try {
+
+            super.open(parameters);
+            logger.info("begin init");
+            IpUtils.loadIpLook();
+            logger.info("init over");
+
             executorService.scheduleAtFixedRate(() -> thresholdRangeMap = ParseStaticThreshold.createStaticThreshold(), 0,
                     CommonConfig.STATIC_THRESHOLD_SCHEDULE_MINUTES, TimeUnit.MINUTES);
 
@@ -58,56 +64,62 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
     }
 
     @Override
-    public DosEventLog map(DosSketchLog value) {
+    public void processElement(DosSketchLog value, ReadOnlyContext ctx, Collector<DosEventLog> out) {
         DosEventLog finalResult = null;
         try {
             String destinationIp = value.getDestination_ip();
+            int vsysId = value.getVsys_id();
+            String key = destinationIp + "-" + vsysId;
             String attackType = value.getAttack_type();
             IPAddress destinationIpAddress = new IPAddressString(destinationIp).getAddress();
-            DosDetectionThreshold threshold = thresholdRangeMap.getOrDefault(attackType, TreeRangeMap.create()).get(destinationIpAddress);
-            logger.debug("当前判断IP：{}, 类型: {}", destinationIp, attackType);
-            if (threshold == null && baselineMap.containsKey(destinationIp)) {
-                finalResult = getDosEventLogByBaseline(value);
-            } else if (threshold == null && !baselineMap.containsKey(destinationIp)) {
+
+            DosDetectionThreshold threshold = null;
+            if (thresholdRangeMap.containsKey(vsysId)){
+                threshold = thresholdRangeMap.get(vsysId).getOrDefault(attackType, TreeRangeMap.create()).get(destinationIpAddress);
+            }
+
+            logger.debug("当前判断IP：{}, 类型: {}", key, attackType);
+            if (threshold == null && baselineMap.containsKey(key)) {
+                finalResult = getDosEventLogByBaseline(value,key);
+            } else if (threshold == null && !baselineMap.containsKey(key)) {
                 finalResult = getDosEventLogBySensitivityThreshold(value);
             } else if (threshold != null) {
                 finalResult = getDosEventLogByStaticThreshold(value, threshold);
             } else {
-                logger.debug("未获取到当前server IP：{} 类型 {} 静态阈值 和 baseline", destinationIp, attackType);
+                logger.debug("未获取到当前server IP：{} 类型 {} 静态阈值 和 baseline", key, attackType);
             }
 
         } catch (Exception e) {
             logger.error("判定失败\n {} \n{}", value, e);
         }
-        return finalResult;
+
+        if (finalResult != null){
+            out.collect(finalResult);
+        }
+    }
+
+    @Override
+    public void processBroadcastElement(Map<String, byte[]> value, Context ctx, Collector<DosEventLog> out) throws Exception {
+        IpUtils.updateIpLook(value);
     }
 
     private DosEventLog getDosEventLogBySensitivityThreshold(DosSketchLog value) {
-        DosEventLog result = null;
         long sketchSessions = value.getSketch_sessions();
-        if (sketchSessions > NacosUtils.getIntProperty("static.sensitivity.threshold")) {
-            long diff = sketchSessions - NacosUtils.getIntProperty("static.sensitivity.threshold");
-            result = getDosEventLog(value, NacosUtils.getIntProperty("static.sensitivity.threshold"), diff, SENSITIVITY_CONDITION_TYPE, SESSIONS_TAG);
-            result.setSeverity(Severity.MAJOR.severity);
-        }
-        return result;
+        Integer staticSensitivityThreshold = NacosUtils.getIntProperty("static.sensitivity.threshold");
+        long diff = sketchSessions - staticSensitivityThreshold;
+        return getDosEventLog(value, staticSensitivityThreshold, diff, SENSITIVITY_CONDITION_TYPE, SESSIONS_TAG);
     }
 
-    private DosEventLog getDosEventLogByBaseline(DosSketchLog value) {
-        DosEventLog result = null;
-        String destinationIp = value.getDestination_ip();
+    private DosEventLog getDosEventLogByBaseline(DosSketchLog value,String key) {
         String attackType = value.getAttack_type();
         long sketchSessions = value.getSketch_sessions();
-        if (sketchSessions > NacosUtils.getIntProperty("static.sensitivity.threshold")) {
-            DosBaselineThreshold dosBaselineThreshold = baselineMap.get(destinationIp).get(attackType);
-            Integer base = getBaseValue(dosBaselineThreshold, value);
-            long diff = sketchSessions - base;
-            result = getDosEventLog(value, base, diff, BASELINE_CONDITION_TYPE, SESSIONS_TAG);
-        }
-        return result;
+        DosBaselineThreshold dosBaselineThreshold = baselineMap.get(key).get(attackType);
+        Integer base = getBaseValue(dosBaselineThreshold, value);
+        long diff = sketchSessions - base;
+        return getDosEventLog(value, base, diff, BASELINE_CONDITION_TYPE, SESSIONS_TAG);
     }
 
-    private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, DosDetectionThreshold threshold) {
+    private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, DosDetectionThreshold threshold) throws CloneNotSupportedException {
         long base = threshold.getSessionsPerSec();
         long diff = value.getSketch_sessions() - base;
         DosEventLog result = getDosEventLog(value, base, diff, STATIC_CONDITION_TYPE, SESSIONS_TAG);
@@ -121,6 +133,21 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
                 result = getDosEventLog(value, base, diff, STATIC_CONDITION_TYPE, BITS_TAG);
             }
         }
+        /*
+        ArrayList<DosEventLog> dosEventLogs = new ArrayList<>();
+        if (result != null){
+            dosEventLogs.add(result);
+            Integer[] superiorIds = threshold.getSuperiorIds();
+            if (superiorIds != null && superiorIds.length > 0){
+                for (Integer integer:superiorIds){
+                    DosEventLog clone = (DosEventLog) result.clone();
+                    clone.setVsys_id(integer);
+                    clone.setLog_id(SnowflakeId.generateId());
+                    dosEventLogs.add(clone);
+                }
+            }
+        }
+        */
         return result;
     }
 
@@ -131,11 +158,17 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
         if (diff > 0 && base != 0) {
             double percent = getDiffPercent(diff, base);
             Severity severity = judgeSeverity(percent);
+            Integer staticSensitivityThreshold = NacosUtils.getIntProperty("static.sensitivity.threshold");
             if (severity != Severity.NORMAL) {
                 if (type == BASELINE_CONDITION_TYPE && percent < NacosUtils.getDoubleProperty("baseline.sensitivity.threshold")) {
                     logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过基线敏感阈值,日志详情\n{}", destinationIp, attackType, base, percent, value);
-                } else {
+                }else if ((type == BASELINE_CONDITION_TYPE || type == SENSITIVITY_CONDITION_TYPE) && value.getSketch_sessions() < staticSensitivityThreshold){
+                    logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过静态敏感阈值,日志详情\n{}",destinationIp, attackType, base, percent, value);
+                }else {
                     result = getResult(value, base, severity, percent+1, type, tag);
+                    if (type == SENSITIVITY_CONDITION_TYPE){
+                        result.setSeverity(Severity.MAJOR.severity);
+                    }
                     logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,基于{}:{}检测,日志详情\n {}", destinationIp,attackType,base,percent,type,tag,result);
                 }
             } else {
@@ -148,6 +181,7 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
     private DosEventLog getResult(DosSketchLog value, long base, Severity severity, double percent, int type, String tag) {
         DosEventLog dosEventLog = new DosEventLog();
         dosEventLog.setLog_id(SnowflakeId.generateId());
+        dosEventLog.setVsys_id(value.getVsys_id());
         dosEventLog.setStart_time(value.getSketch_start_time());
         dosEventLog.setEnd_time(value.getSketch_start_time() + value.getSketch_duration());
         dosEventLog.setAttack_type(value.getAttack_type());
@@ -192,22 +226,16 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
     private String getConditions(String percent, long base, long sessions, int type, String tag) {
         switch (type) {
             case STATIC_CONDITION_TYPE:
-                return new StrBuilder()
-                        .append("Rate > ")
-                        .append(base).append(" ")
-                        .append(tag).append("/s")
-                        .toString();
+                return "Rate > " +
+                        base + " " +
+                        tag + "/s";
             case BASELINE_CONDITION_TYPE:
-                return new StrBuilder()
-                        .append(tag).append(" > ")
-                        .append(percent).append(" of baseline")
-                        .toString();
+                return tag + " > " +
+                        percent + " of baseline";
             case SENSITIVITY_CONDITION_TYPE:
-                return new StrBuilder()
-                        .append(sessions).append(" ")
-                        .append(tag).append("/s Unusually high ")
-                        .append(StringUtils.capitalize(tag))
-                        .toString();
+                return String.valueOf(sessions) + " " +
+                        tag + "/s Unusually high " +
+                        StringUtils.capitalize(tag);
             default:
                 throw new IllegalArgumentException("Illegal Argument type:" + type + ", known types = [1,2,3]");
         }

src/main/java/com/zdjizhi/etl/EtlProcessFunction.java

@@ -1,15 +1,15 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosSketchLog;
-import org.apache.commons.lang.StringUtils;
-import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.tuple.Tuple6;
 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.util.HashSet;
 
@@ -18,14 +18,15 @@ import static com.zdjizhi.sink.OutputStreamSink.outputTag;
 /**
  * @author 94976
  */
-public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosSketchLog, Tuple2<String,String>, TimeWindow> {
+public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosSketchLog, Tuple3<String,String,Integer>, TimeWindow> {
 
-    private static final Logger logger = LoggerFactory.getLogger(EtlProcessFunction.class);
+//    private static final Logger logger = LoggerFactory.getLogger(EtlProcessFunction.class);
+    private static final Log logger = LogFactory.get();
     private static final String EMPTY_SOURCE_IP_IPV4 = "0.0.0.0";
     private static final String EMPTY_SOURCE_IP_IPV6 = "::";
 
     @Override
-    public void process(Tuple2<String, String> keys,
+    public void process(Tuple3<String,String,Integer> keys,
                         Context context, Iterable<DosSketchLog> elements,
                         Collector<DosSketchLog> out) {
         DosSketchLog middleResult = getMiddleResult(keys, elements);
@@ -40,7 +41,7 @@ public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosS
         }
     }
 
-    private DosSketchLog getMiddleResult(Tuple2<String, String> keys,Iterable<DosSketchLog> elements){
+    private DosSketchLog getMiddleResult(Tuple3<String,String,Integer> keys,Iterable<DosSketchLog> elements){
 
         DosSketchLog midResuleLog = new DosSketchLog();
         Tuple6<Long, Long, Long,String,Long,Long> values = sketchAggregate(elements);
@@ -48,6 +49,7 @@ public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosS
             if (values != null){
                 midResuleLog.setAttack_type(keys.f0);
                 midResuleLog.setDestination_ip(keys.f1);
+                midResuleLog.setVsys_id(keys.f2);
                 midResuleLog.setSketch_start_time(values.f4);
                 midResuleLog.setSketch_duration(values.f5);
                 midResuleLog.setSource_ip(values.f3);

src/main/java/com/zdjizhi/etl/ParseBaselineThreshold.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosBaselineThreshold;
 import com.zdjizhi.utils.DateUtils;
@@ -9,15 +11,14 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.*;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.*;
 
 public class ParseBaselineThreshold {
 
-    private static final Logger logger = LoggerFactory.getLogger(ParseBaselineThreshold.class);
+//    private static final Logger logger = LoggerFactory.getLogger(ParseBaselineThreshold.class);
+    private static final Log logger = LogFactory.get();
     private static ArrayList<String> floodTypeList = new ArrayList<>();
 
     private static Table table = null;

src/main/java/com/zdjizhi/etl/ParseSketchLog.java

@@ -1,25 +1,15 @@
 package com.zdjizhi.etl;
 
-import com.alibaba.nacos.api.PropertyKeyConst;
 import com.fasterxml.jackson.databind.JavaType;
 import com.zdjizhi.common.CommonConfig;
-import com.zdjizhi.common.CustomFile;
 import com.zdjizhi.common.DosSketchLog;
-import com.zdjizhi.function.BroadcastProcessFunc;
 import com.zdjizhi.source.DosSketchSource;
 import com.zdjizhi.utils.FlinkEnvironmentUtils;
 import com.zdjizhi.utils.JsonMapper;
 import com.zdjizhi.utils.StringUtil;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.state.MapStateDescriptor;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeinfo.Types;
-import org.apache.flink.streaming.api.datastream.BroadcastConnectedStream;
-import org.apache.flink.streaming.api.datastream.BroadcastStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction;
 import org.apache.flink.util.Collector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -43,28 +33,7 @@ public class ParseSketchLog {
     }
 
     private static SingleOutputStreamOperator<DosSketchLog> flatSketchSource(){
-
-        DataStreamSource<Map<String, byte[]>> broadcastSource=null;
-        Properties nacosProperties = new Properties();
-
-        nacosProperties.put(PropertyKeyConst.SERVER_ADDR,CommonConfig.NACOS_SERVER_ADDR);
-        nacosProperties.setProperty(PropertyKeyConst.USERNAME, CommonConfig.NACOS_USERNAME);
-        nacosProperties.setProperty(PropertyKeyConst.PASSWORD, CommonConfig.NACOS_PASSWORD);
-
-        if ("CLUSTER".equals(CommonConfig.CLUSTER_OR_SINGLE)){
-            broadcastSource = DosSketchSource.broadcastSource(nacosProperties,CommonConfig.HDFS_PATH);
-        }else {
-            broadcastSource= DosSketchSource.singleBroadcastSource(nacosProperties);
-        }
-
-        MapStateDescriptor<String,Map> descriptor =
-                new MapStateDescriptor<>("descriptorTest", Types.STRING, TypeInformation.of(Map.class));
-
-        BroadcastStream<Map<String, byte[]>> broadcast = broadcastSource.broadcast(descriptor);
-//        BroadcastConnectedStream<String, List<CustomFile>> connect = DosSketchSource.createDosSketchSource().connect(broadcast);
-        return DosSketchSource.createDosSketchSource()
-                .connect(broadcast).process(new BroadcastProcessFunc());
-//                .flatMap(new FlatSketchLog());
+        return DosSketchSource.createDosSketchSource().flatMap(new FlatSketchLog());
     }
 
     private static WatermarkStrategy<DosSketchLog> createWatermarkStrategy(){
@@ -82,12 +51,14 @@ public class ParseSketchLog {
                     long sketchStartTime = Long.parseLong(sketchSource.get("sketch_start_time").toString());
                     long sketchDuration = Long.parseLong(sketchSource.get("sketch_duration").toString());
                     String attackType = sketchSource.get("attack_type").toString();
+                    int vsysId = Integer.parseInt(sketchSource.getOrDefault("common_vsys_id", 1).toString());
                     ArrayList<HashMap<String, Object>> reportIpList = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(sketchSource.get("report_ip_list")), listType);
                     for (HashMap<String, Object> obj : reportIpList) {
                         DosSketchLog dosSketchLog = new DosSketchLog();
                         dosSketchLog.setSketch_start_time(sketchStartTime);
                         dosSketchLog.setSketch_duration(sketchDuration);
                         dosSketchLog.setAttack_type(attackType);
+                        dosSketchLog.setVsys_id(vsysId);
                         String sourceIp = obj.get("source_ip").toString();
                         String destinationIp = obj.get("destination_ip").toString();
                         long sketchSessions = Long.parseLong(obj.get("sketch_sessions").toString());

src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.fasterxml.jackson.databind.JavaType;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosDetectionThreshold;
@@ -13,8 +15,6 @@ import org.apache.flink.shaded.guava18.com.google.common.collect.Range;
 import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.message.BasicHeader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.net.URISyntaxException;
 import java.util.ArrayList;
@@ -25,7 +25,8 @@ import java.util.Map;
  * @author wlh
  */
 public class ParseStaticThreshold {
-    private static Logger logger = LoggerFactory.getLogger(ParseStaticThreshold.class);
+//    private static Logger logger = LoggerFactory.getLogger(ParseStaticThreshold.class);
+    private static final Log logger = LogFactory.get();
     private static String encryptpwd;
 
     private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();
@@ -112,7 +113,8 @@ public class ParseStaticThreshold {
             URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
             HashMap<String, Object> parms = new HashMap<>();
             parms.put("pageSize", -1);
-            parms.put("orderBy", "vsysId desc");
+//            parms.put("orderBy", "vsysId desc");
+            parms.put("type", 1);
             HttpClientUtils.setUrlWithParams(uriBuilder, CommonConfig.BIFANG_SERVER_POLICY_VSYSID_PATH, parms);
             String token = NacosUtils.getStringProperty("bifang.server.token");
             if (!HttpClientUtils.ERROR_MESSAGE.equals(token)) {
@@ -128,7 +130,7 @@ public class ParseStaticThreshold {
                         Object list = data.get("list");
                         if (list != null) {
                             vsysIdList = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(list), vsysIDType);
-                            logger.info("获取到vsysId{}条", vsysIdList.size());
+                            logger.info("获取到vsysId {}条", vsysIdList.size());
                         } else {
                             logger.warn("vsysIdList为空");
                         }
@@ -149,49 +151,54 @@ public class ParseStaticThreshold {
      * @return thresholds
      */
     private static ArrayList<DosDetectionThreshold> getDosDetectionThreshold() {
-        ArrayList<DosDetectionThreshold> thresholds = null;
-//        ArrayList<DosVsysId> vsysId = getVsysId();
+        ArrayList<DosDetectionThreshold> vsysThresholds = new ArrayList<>();
+        ArrayList<DosVsysId> vsysIds = getVsysId();
         try {
-//            if (vsysId != null){
-//             for (DosVsysId dosVsysId : vsysId) {
-            URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
-            HashMap<String, Object> parms = new HashMap<>();
-            parms.put("pageSize", -1);
-            parms.put("orderBy", "profileId asc");
-            parms.put("isValid", 1);
-//                     parms.put("vsysId", dosVsysId.getVsysId());
-            parms.put("vsysId", 1);
-            HttpClientUtils.setUrlWithParams(uriBuilder, CommonConfig.BIFANG_SERVER_POLICY_THRESHOLD_PATH, parms);
-            String token = NacosUtils.getStringProperty("bifang.server.token");
-            if (!HttpClientUtils.ERROR_MESSAGE.equals(token)) {
-                BasicHeader authorization = new BasicHeader("Authorization", token);
-                BasicHeader authorization1 = new BasicHeader("Content-Type", "application/x-www-form-urlencoded");
-                String resposeJsonStr = HttpClientUtils.httpGet(uriBuilder.build(), authorization, authorization1);
-                if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)) {
-                    HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
-                    boolean success = (boolean) resposeMap.get("success");
-                    String msg = resposeMap.get("msg").toString();
-                    if (success) {
-                        HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
-                        Object list = data.get("list");
-                        if (list != null) {
-                            thresholds = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(list), thresholdType);
-                            logger.info("获取到静态阈值配置{}条", thresholds.size());
-                        } else {
-                            logger.warn("静态阈值配置为空");
+            if (vsysIds != null) {
+                for (DosVsysId dosVsysId : vsysIds) {
+                    Integer vsysId = dosVsysId.getId() == null ? 1 : dosVsysId.getId();
+                    Integer[] superiorIds = dosVsysId.getSuperiorIds();
+                    URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
+                    HashMap<String, Object> parms = new HashMap<>();
+                    parms.put("pageSize", -1);
+                    parms.put("orderBy", "profileId asc");
+                    parms.put("isValid", 1);
+                    parms.put("vsysId", vsysId);
+                    HttpClientUtils.setUrlWithParams(uriBuilder, CommonConfig.BIFANG_SERVER_POLICY_THRESHOLD_PATH, parms);
+                    String token = NacosUtils.getStringProperty("bifang.server.token");
+                    if (!HttpClientUtils.ERROR_MESSAGE.equals(token)) {
+                        BasicHeader authorization = new BasicHeader("Authorization", token);
+                        BasicHeader authorization1 = new BasicHeader("Content-Type", "application/x-www-form-urlencoded");
+                        String resposeJsonStr = HttpClientUtils.httpGet(uriBuilder.build(), authorization, authorization1);
+                        if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)) {
+                            HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
+                            boolean success = (boolean) resposeMap.get("success");
+                            String msg = resposeMap.get("msg").toString();
+                            if (success) {
+                                HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
+                                Object list = data.get("list");
+                                if (list != null) {
+                                    ArrayList<DosDetectionThreshold> thresholds = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(list), thresholdType);
+                                    for (DosDetectionThreshold dosDetectionThreshold : thresholds) {
+                                        dosDetectionThreshold.setSuperiorIds(superiorIds);
+                                        vsysThresholds.add(dosDetectionThreshold);
+                                    }
+                                    logger.info("获取到vsys id是{}静态阈值配置{}条", vsysId, thresholds.size());
+                                } else {
+                                    logger.warn("静态阈值配置为空");
+                                }
+                            } else {
+                                logger.error(msg);
+                            }
                         }
-                    } else {
-                        logger.error(msg);
                     }
                 }
             }
-//             }
-//            }
         } catch (Exception e) {
             logger.error("获取静态阈值配置失败,请检查bifang服务或登录配置信息 ", e);
         }
 
-        return thresholds;
+        return vsysThresholds;
     }
 
     /**
@@ -199,14 +206,17 @@ public class ParseStaticThreshold {
      *
      * @return threshold RangeMap
      */
-    static HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>> createStaticThreshold() {
-        HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>> thresholdRangeMap = new HashMap<>(4);
+    static HashMap<Integer, HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> createStaticThreshold() {
+        HashMap<Integer, HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> thresholdRangeMap = new HashMap<>(4);
         try {
             ArrayList<DosDetectionThreshold> dosDetectionThreshold = getDosDetectionThreshold();
             if (dosDetectionThreshold != null && !dosDetectionThreshold.isEmpty()) {
                 for (DosDetectionThreshold threshold : dosDetectionThreshold) {
                     String attackType = threshold.getAttackType();
-                    TreeRangeMap<IPAddress, DosDetectionThreshold> treeRangeMap = thresholdRangeMap.getOrDefault(attackType, TreeRangeMap.create());
+                    int vsysId = threshold.getVsysId();
+                    HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>> rangeMap = thresholdRangeMap.getOrDefault(vsysId, new HashMap<>());
+
+                    TreeRangeMap<IPAddress, DosDetectionThreshold> treeRangeMap = rangeMap.getOrDefault(attackType, TreeRangeMap.create());
                     ArrayList<String> serverIpList = threshold.getServerIpList();
                     for (String sip : serverIpList) {
                         IPAddressString ipAddressString = new IPAddressString(sip);
@@ -239,7 +249,8 @@ public class ParseStaticThreshold {
                             }
                         }
                     }
-                    thresholdRangeMap.put(attackType, treeRangeMap);
+                    rangeMap.put(attackType, treeRangeMap);
+                    thresholdRangeMap.put(vsysId, rangeMap);
                 }
             }
         } catch (Exception e) {
@@ -249,22 +260,27 @@ public class ParseStaticThreshold {
     }
 
     public static void main(String[] args) {
+        /*
         ArrayList<DosDetectionThreshold> dosDetectionThreshold = getDosDetectionThreshold();
-        dosDetectionThreshold.forEach(System.out::println);
-
-
+//        dosDetectionThreshold.forEach(System.out::println);
+        getVsysId().forEach(System.out::println);
         System.out.println("------------------------");
-        HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>> staticThreshold = createStaticThreshold();
+        */
+        HashMap<Integer, HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> staticThreshold = createStaticThreshold();
 
         System.out.println("------------------------");
 
-        for (String type : staticThreshold.keySet()) {
-            Map<Range<IPAddress>, DosDetectionThreshold> asMapOfRanges = staticThreshold.get(type).asMapOfRanges();
-            for (Range<IPAddress> range : asMapOfRanges.keySet()) {
-                DosDetectionThreshold threshold = asMapOfRanges.get(range);
-                System.out.println(type + "---" + range + "---" + threshold);
+        for (Integer integer : staticThreshold.keySet()) {
+            HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>> stringTreeRangeMapHashMap = staticThreshold.get(integer);
+            for (String type : stringTreeRangeMapHashMap.keySet()) {
+                Map<Range<IPAddress>, DosDetectionThreshold> asMapOfRanges = stringTreeRangeMapHashMap.get(type).asMapOfRanges();
+                for (Range<IPAddress> range : asMapOfRanges.keySet()) {
+                    DosDetectionThreshold threshold = asMapOfRanges.get(range);
+                    System.out.println(integer + "---" + type + "---" + range + "---" + threshold);
+                }
+                System.out.println("------------------------");
             }
-            System.out.println("------------------------");
+
         }
 //        String s = loginBifangServer();
 //        System.out.println(s);

src/main/java/com/zdjizhi/etl/TrafficServerIpMetrics.java

@@ -1,15 +1,15 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosMetricsLog;
 import com.zdjizhi.common.DosSketchLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 
 class TrafficServerIpMetrics {
 
-    private static final Logger logger = LoggerFactory.getLogger(TrafficServerIpMetrics.class);
+//    private static final Logger logger = LoggerFactory.getLogger(TrafficServerIpMetrics.class);
+    private static final Log logger = LogFactory.get();
 
     static DosMetricsLog getOutputMetric(DosSketchLog midResuleLog) {
         DosMetricsLog dosMetricsLog = new DosMetricsLog();
@@ -19,6 +19,7 @@ class TrafficServerIpMetrics {
         dosMetricsLog.setSession_rate(midResuleLog.getSketch_sessions());
         dosMetricsLog.setPacket_rate(midResuleLog.getSketch_packets());
         dosMetricsLog.setBit_rate(midResuleLog.getSketch_bytes());
+        dosMetricsLog.setVsys_id(midResuleLog.getVsys_id());
         dosMetricsLog.setPartition_num(getPartitionNumByIp(midResuleLog.getDestination_ip()));
         logger.debug("metric 结果已加载：{}",dosMetricsLog.toString());
         return dosMetricsLog;

src/main/java/com/zdjizhi/function/BroadcastProcessFunc.java

@@ -1,76 +0,0 @@
-package com.zdjizhi.function;
-
-import com.fasterxml.jackson.databind.JavaType;
-import com.zdjizhi.common.CustomFile;
-import com.zdjizhi.common.DosSketchLog;
-import com.zdjizhi.etl.ParseSketchLog;
-import com.zdjizhi.utils.IpUtils;
-import com.zdjizhi.utils.JsonMapper;
-import com.zdjizhi.utils.StringUtil;
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction;
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class BroadcastProcessFunc extends BroadcastProcessFunction<String, Map<String, byte[]>, DosSketchLog> {
-    private static Logger logger = LoggerFactory.getLogger(ParseSketchLog.class);
-    private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();
-    private static JavaType hashmapJsonType = jsonMapperInstance.createCollectionType(HashMap.class, String.class, Object.class);
-    private static JavaType listType = jsonMapperInstance.createCollectionType(ArrayList.class, HashMap.class);
-    @Override
-    public void open(Configuration parameters) throws Exception {
-        super.open(parameters);
-        System.out.println("begin init");
-        IpUtils.loadIpLook();
-        System.out.println("init over");
-    }
-
-    @Override
-    public void processElement(String value, ReadOnlyContext ctx, Collector<DosSketchLog> out) throws Exception {
-        try {
-            if (StringUtil.isNotBlank(value)){
-                HashMap<String, Object> sketchSource = jsonMapperInstance.fromJson(value, hashmapJsonType);
-                long sketchStartTime = Long.parseLong(sketchSource.get("sketch_start_time").toString());
-                long sketchDuration = Long.parseLong(sketchSource.get("sketch_duration").toString());
-                String attackType = sketchSource.get("attack_type").toString();
-                ArrayList<HashMap<String, Object>> reportIpList = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(sketchSource.get("report_ip_list")), listType);
-                for (HashMap<String, Object> obj : reportIpList) {
-                    DosSketchLog dosSketchLog = new DosSketchLog();
-                    dosSketchLog.setSketch_start_time(sketchStartTime);
-                    dosSketchLog.setSketch_duration(sketchDuration);
-                    dosSketchLog.setAttack_type(attackType);
-                    String sourceIp = obj.get("source_ip").toString();
-                    String destinationIp = obj.get("destination_ip").toString();
-                    long sketchSessions = Long.parseLong(obj.get("sketch_sessions").toString());
-                    long sketchPackets = Long.parseLong(obj.get("sketch_packets").toString());
-                    long sketchBytes = Long.parseLong(obj.get("sketch_bytes").toString());
-                    dosSketchLog.setSource_ip(sourceIp);
-                    dosSketchLog.setDestination_ip(destinationIp);
-                    dosSketchLog.setSketch_sessions(sketchSessions);
-                    dosSketchLog.setSketch_packets(sketchPackets);
-                    dosSketchLog.setSketch_bytes(sketchBytes);
-                    out.collect(dosSketchLog);
-                    logger.debug("数据解析成功：{}",dosSketchLog.toString());
-                }
-            }
-        } catch (Exception e) {
-            logger.error("数据解析错误：{} \n{}",value,e);
-        }
-    }
-
-    @Override
-    public void processBroadcastElement(Map<String, byte[]> value, Context ctx, Collector<DosSketchLog> out) throws Exception {
-        IpUtils.updateIpLook(value);
-    }
-
-}
-
-
-

src/main/java/com/zdjizhi/sink/OutputStreamSink.java

@@ -1,5 +1,8 @@
 package com.zdjizhi.sink;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.alibaba.nacos.api.PropertyKeyConst;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosEventLog;
 import com.zdjizhi.common.DosMetricsLog;
@@ -7,21 +10,27 @@ import com.zdjizhi.common.DosSketchLog;
 import com.zdjizhi.etl.DosDetection;
 import com.zdjizhi.etl.EtlProcessFunction;
 import com.zdjizhi.etl.ParseSketchLog;
+import com.zdjizhi.source.DosSketchSource;
 import com.zdjizhi.utils.FlinkEnvironmentUtils;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeinfo.Types;
 import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.streaming.api.datastream.*;
 import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.util.OutputTag;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Properties;
 
 /**
  * @author 94976
  */
 public class OutputStreamSink {
-    private static final Logger logger = LoggerFactory.getLogger(OutputStreamSink.class);
+//    private static final Logger logger = LoggerFactory.getLogger(OutputStreamSink.class);
+    private static final Log logger = LogFactory.get();
 
     public static OutputTag<DosMetricsLog> outputTag = new OutputTag<DosMetricsLog>("traffic server ip metrics"){};
 
@@ -37,7 +46,29 @@ public class OutputStreamSink {
     }
 
     private static SingleOutputStreamOperator<DosEventLog> getEventSinkStream(SingleOutputStreamOperator<DosSketchLog> middleStream){
-        return middleStream.map(new DosDetection()).setParallelism(CommonConfig.FLINK_DETECTION_MAP_PARALLELISM);
+        DataStreamSource<Map<String, byte[]>> broadcastSource=null;
+        Properties nacosProperties = new Properties();
+
+        nacosProperties.put(PropertyKeyConst.SERVER_ADDR,CommonConfig.NACOS_SERVER_ADDR);
+        nacosProperties.setProperty(PropertyKeyConst.USERNAME, CommonConfig.NACOS_USERNAME);
+        nacosProperties.setProperty(PropertyKeyConst.PASSWORD, CommonConfig.NACOS_PASSWORD);
+        nacosProperties.setProperty(PropertyKeyConst.NAMESPACE, CommonConfig.NACOS_NAMESPACE);
+
+        if ("CLUSTER".equals(CommonConfig.CLUSTER_OR_SINGLE)){
+            broadcastSource = DosSketchSource.broadcastSource(nacosProperties,CommonConfig.HDFS_PATH);
+        }else {
+            broadcastSource= DosSketchSource.singleBroadcastSource(nacosProperties);
+        }
+
+        MapStateDescriptor<String,Map> descriptor =
+                new MapStateDescriptor<>("descriptorTest", Types.STRING, TypeInformation.of(Map.class));
+
+        BroadcastStream<Map<String, byte[]>> broadcast = broadcastSource.broadcast(descriptor);
+
+        return middleStream
+                .connect(broadcast)
+                .process(new DosDetection()).setParallelism(CommonConfig.FLINK_DETECTION_MAP_PARALLELISM);
+
     }
 
     private static SingleOutputStreamOperator<DosSketchLog> getMiddleStream(){
@@ -48,12 +79,13 @@ public class OutputStreamSink {
                 .setParallelism(CommonConfig.FLINK_FIRST_AGG_PARALLELISM);
     }
 
-    private static class KeysSelector implements KeySelector<DosSketchLog, Tuple2<String, String>>{
+    private static class KeysSelector implements KeySelector<DosSketchLog, Tuple3<String, String, Integer>>{
         @Override
-        public Tuple2<String, String> getKey(DosSketchLog dosSketchLog){
-            return Tuple2.of(
+        public Tuple3<String, String, Integer> getKey(DosSketchLog dosSketchLog){
+            return Tuple3.of(
                     dosSketchLog.getAttack_type(),
-                    dosSketchLog.getDestination_ip());
+                    dosSketchLog.getDestination_ip(),
+                    dosSketchLog.getVsys_id());
         }
     }
 

src/main/java/com/zdjizhi/source/HttpSource.java

@@ -2,6 +2,8 @@ package com.zdjizhi.source;
 
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.io.file.FileReader;
+import cn.hutool.crypto.digest.DigestUtil;
 import cn.hutool.json.JSONObject;
 import com.alibaba.nacos.api.NacosFactory;
 import com.alibaba.nacos.api.PropertyKeyConst;
@@ -28,10 +30,18 @@ import java.util.concurrent.Executor;
 
 
 public class HttpSource extends RichHttpSourceFunction<Map<String, byte[]>> {
-
     private static final Logger logger = LoggerFactory.getLogger(HttpSource.class);
 
+    private static final int TRY_TIMES = 3;
+
     private static final String EXPR = "$.[?(@.version=='latest' && @.name in ['ip_v4_built_in','ip_v6_built_in','ip_v4_user_defined','ip_v6_user_defined'])].['name','sha256','format','path']";
+//    private static final String EXPR = "$.[?(@.version=='latest' && @.name in ['ip_v4_user_defined'])].['name','sha256','format','path']";
+
+    private static Map<String, String> knowledgeMetaCache = new HashMap<>();
+
+    private static HashMap<String, byte[]> knowledgeFileCache;
+
+    private static HttpClientUtils2 httpClientUtils;
 
     //连接nacos的配置
     private Properties nacosProperties;
@@ -50,11 +60,17 @@ public class HttpSource extends RichHttpSourceFunction<Map<String, byte[]>> {
 
     private ConfigService configService;
 
-//    private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();
-//    private static JavaType listType = jsonMapperInstance.createCollectionType(List.class, KnowledgeLog.class);
-    private static Map<String, String> updateMap = new HashMap<>();
-    private static HashMap<String, byte[]> knowledgeFileCache;
+    private static Header header;
+
+    //运行状态cancel时置为false
     private boolean isRunning = true;
+    //是否下发，默认不发送
+    private boolean isSending = false;
+
+
+
+
+//    private boolean isRunning = true;
 
 
     public HttpSource(Properties nacosProperties, String NACOS_DATA_ID, String NACOS_GROUP, long NACOS_READ_TIMEOUT, String storePath) {
@@ -68,10 +84,12 @@ public class HttpSource extends RichHttpSourceFunction<Map<String, byte[]>> {
     @Override
     public void open(Configuration parameters) throws Exception {
         super.open(parameters);
+        httpClientUtils = new HttpClientUtils2();
         //初始化元数据缓存
-        updateMap = new HashMap<>(16);
+        knowledgeMetaCache = new HashMap<>(16);
         //初始化定位库缓存
         knowledgeFileCache = new HashMap<>(16);
+        header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
         logger.info("连接nacos：" + nacosProperties.getProperty(PropertyKeyConst.SERVER_ADDR));
         configService = NacosFactory.createConfigService(nacosProperties);
     }
@@ -79,13 +97,12 @@ public class HttpSource extends RichHttpSourceFunction<Map<String, byte[]>> {
     public void run(SourceContext ctx) throws Exception {
 //        ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
         String config = configService.getConfig(NACOS_DATA_ID, NACOS_GROUP, NACOS_READ_TIMEOUT);
-        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-        String format = formatter.format(new Date());
-        logger.info(format + "receive config from nacos:" + config);
-        System.out.println(format + "receive config from nacos:" + config);
         if (StringUtil.isNotBlank(config)) {
             ArrayList<Object> metaList = JsonPath.parse(config).read(EXPR);
             loadKnowledge(metaList);
+            if (isSending){
+                ctx.collect(knowledgeFileCache);
+            }
         }
 
 
@@ -108,13 +125,14 @@ public class HttpSource extends RichHttpSourceFunction<Map<String, byte[]>> {
                                         knowledgeJson.getStr("format"));
                                 String sha256 = knowledgeJson.getStr("sha256");
                                 String filePath = knowledgeJson.getStr("path");
-                                if (!sha256.equals(updateMap.get(fileName))) {
-                                    updateMap.put(fileName, sha256);
-                                    updateKnowledge(fileName, filePath);
+                                if (!sha256.equals(knowledgeMetaCache.get(fileName))) {
+                                    knowledgeMetaCache.put(fileName, sha256);
+                                    updateKnowledge(fileName, filePath,sha256);
                                 }
-
                             }
-                            ctx.collect(knowledgeFileCache);
+                            if (isSending){
+                                ctx.collect(knowledgeFileCache);
+                            }
                         }
                     }
 
@@ -126,7 +144,11 @@ public class HttpSource extends RichHttpSourceFunction<Map<String, byte[]>> {
         });
 
         while (isRunning) {
-            Thread.sleep(10000);
+            try {
+                Thread.sleep(10000);
+            }catch (InterruptedException e) {
+                e.printStackTrace();
+            }
         }
 
     }
@@ -141,38 +163,92 @@ public class HttpSource extends RichHttpSourceFunction<Map<String, byte[]>> {
                             knowledgeJson.getStr("format"));
                     String sha256 = knowledgeJson.getStr("sha256");
                     String filePath = knowledgeJson.getStr("path");
-                    Header header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
-                    HttpClientUtils2 httpClientUtils = new HttpClientUtils2();
-                    inputStream = httpClientUtils.httpGetInputStream(filePath, 3000, header);
-                    updateMap.put(fileName, sha256);
-                    knowledgeFileCache.put(fileName, IOUtils.toByteArray(inputStream));
+                    byte[] localFileByte = getLocalFile(fileName);
+                    String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
+                    if (sha256.equals(localFileSha256Hex)){
+                        logger.info("本地文件{}的sha256为:{} ,Nacos内记录为:{} ,sha256相等", fileName, localFileSha256Hex, sha256);
+                        knowledgeMetaCache.put(fileName, sha256);
+//                        knowledgeFileCache.put(fileName, IOUtils.toByteArray(inputStream));
+                        knowledgeFileCache.put(fileName, localFileByte);
+                    }else {
+                        logger.info("本地文件{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等，更新本地文件及缓存", fileName, localFileSha256Hex, sha256);
+                        updateKnowledge(fileName,filePath,sha256);
+                    }
+
+                }
+            }
+//        } catch (IOException ioException) {
+        } catch (Exception ioException) {
+            ioException.printStackTrace();
+        } finally {
+            IOUtils.closeQuietly(inputStream);
+        }
+    }
+
+
+    private void updateKnowledge(String fileName, String filePath,String sha256) {
+        InputStream inputStream = null;
+//        FileOutputStream outputStream = null;
+        int retryNum = 0;
+        try {
+            while (retryNum < TRY_TIMES){
+                inputStream = httpClientUtils.httpGetInputStream(filePath, 90000, header);
+                if (inputStream !=null){
+                    byte[] downloadBytes = IOUtils.toByteArray(inputStream);
+                    if (downloadBytes.length>0){
+                        String downloadFileSha256Hex = DigestUtil.sha256Hex(downloadBytes);
+                        if (sha256.equals(downloadFileSha256Hex)){
+                            logger.info("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256相等", fileName, sha256);
+//                            HdfsUtils.uploadFileByBytes(CommonConfig.HDFS_PATH + fileName, downloadBytes);
+                            knowledgeMetaCache.put(fileName,sha256);
+                            knowledgeFileCache.put(fileName, downloadBytes);
+                            updateLocalFile(fileName);
+                            retryNum = TRY_TIMES;
+
+                            isSending = true;
+                        }else {
+                            logger.error("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等 开始第{}次重试下载文件", fileName, downloadFileSha256Hex, sha256, retryNum);
+                            retryNum++;
+                        }
+                    }
                 }
             }
         } catch (IOException ioException) {
             ioException.printStackTrace();
         } finally {
             IOUtils.closeQuietly(inputStream);
+//            IOUtils.closeQuietly(outputStream);
         }
     }
 
 
-    private void updateKnowledge(String fileName, String filePath) {
-        InputStream inputStream = null;
+    private void updateLocalFile(String fileName) {
         FileOutputStream outputStream = null;
         try {
-            Header header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
-            HttpClientUtils2 httpClientUtils = new HttpClientUtils2();
-            inputStream = httpClientUtils.httpGetInputStream(filePath, 3000, header);
-            byte[] bytes = IOUtils.toByteArray(inputStream);
-            HdfsUtils.uploadFileByBytes(CommonConfig.HDFS_PATH + fileName, bytes);
-            knowledgeFileCache.put(fileName, bytes);
-        } catch (IOException ioException) {
-            ioException.printStackTrace();
+            HdfsUtils.uploadFileByBytes(CommonConfig.HDFS_PATH + fileName, knowledgeFileCache.get(fileName));
+        } catch (IOException ioe) {
+            logger.error("更新本地文件{}时发生IO异常,异常信息为:", fileName, ioe.getMessage());
+            ioe.printStackTrace();
+        } catch (RuntimeException e) {
+            logger.error("更新本地文件{}时发生异常,异常信息为:", fileName, e.getMessage());
+            e.printStackTrace();
         } finally {
-            IOUtils.closeQuietly(inputStream);
             IOUtils.closeQuietly(outputStream);
         }
+
     }
+
+    private static byte[] getLocalFile(String name) {
+        byte[] fileBytes = null;
+        try {
+            fileBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + name) ;
+        } catch (RuntimeException | IOException e) {
+            logger.error("IpLookupUtils download MMDB files error, message is:" + e.getMessage());
+            e.printStackTrace();
+        }
+        return fileBytes;
+    }
+
     @Override
     public void cancel() {
         this.isRunning = false;

src/main/java/com/zdjizhi/source/SingleHttpSource.java

@@ -2,6 +2,8 @@ package com.zdjizhi.source;
 
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.io.file.FileReader;
+import cn.hutool.crypto.digest.DigestUtil;
 import cn.hutool.json.JSONObject;
 import com.alibaba.nacos.api.NacosFactory;
 import com.alibaba.nacos.api.PropertyKeyConst;
@@ -23,13 +25,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.concurrent.Executor;
 
 public class SingleHttpSource extends RichHttpSourceFunction<Map<String, byte[]>> {
 
-    private static final Logger logger = LoggerFactory.getLogger(HttpSource.class);
-    private static HashMap<String, byte[]> knowledgeFileCache;
+    private static final Logger logger = LoggerFactory.getLogger(SingleHttpSource.class);
 
     private Properties nacosProperties;
 
@@ -41,16 +43,23 @@ public class SingleHttpSource extends RichHttpSourceFunction<Map<String, byte[]>
 
     private static String STORE_PATH;
 
+    private static HttpClientUtils2 httpClientUtils ;
+
     private ConfigService configService;
 
-//    private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();
-//    private static JavaType listType = jsonMapperInstance.createCollectionType(List.class, KnowledgeLog.class);
+    private static Header header;
+
     private static final String EXPR = "$.[?(@.version=='latest' && @.name in ['ip_v4_built_in','ip_v6_built_in','ip_v4_user_defined','ip_v6_user_defined'])].['name','sha256','format','path']";
 
 
-    private static Map<String, String> updateMap = new HashMap<>();
+    private static Map<String, String> knowledgeMetaCache = new HashMap<>();
 
+    private static HashMap<String, byte[]> knowledgeFileCache;
     private boolean isRunning = true;
+    //是否下发，默认不发送
+    private boolean isSending = false;
+
+    private static final int TRY_TIMES = 3;
 
 
     public SingleHttpSource(Properties nacosProperties, String NACOS_DATA_ID, String NACOS_GROUP, long NACOS_READ_TIMEOUT) {
@@ -65,33 +74,33 @@ public class SingleHttpSource extends RichHttpSourceFunction<Map<String, byte[]>
     @Override
     public void open(Configuration parameters) throws Exception {
         super.open(parameters);
+        httpClientUtils = new HttpClientUtils2();
         logger.info("连接nacos：" + nacosProperties.getProperty(PropertyKeyConst.SERVER_ADDR));
         configService = NacosFactory.createConfigService(nacosProperties);
         //初始化元数据缓存
-        updateMap = new HashMap<>(16);
+        knowledgeMetaCache = new HashMap<>(16);
         //初始化定位库缓存
         knowledgeFileCache = new HashMap<>(16);
+
+        header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
     }
 
     @Override
     public void run(SourceContext ctx) throws Exception {
 //        ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
         String config = configService.getConfig(NACOS_DATA_ID, NACOS_GROUP, NACOS_READ_TIMEOUT);
-//        List<CustomFile> customFiles = new ArrayList<>();
+        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        String format = formatter.format(new Date());
+        logger.info(format + "receive config from nacos:" + config);
+        System.out.println(format + "receive config from nacos:" + config);
         if (StringUtil.isNotBlank(config)) {
             ArrayList<Object> metaList = JsonPath.parse(config).read(EXPR);
             loadKnowledge(metaList);
+            if (isSending){
+                ctx.collect(knowledgeFileCache);
+            }
         }
-//        if (StringUtil.isNotBlank(config)) {
-//            List<KnowledgeLog> knowledgeLogListList = jsonMapperInstance.fromJson(config, listType);
-//            if (knowledgeLogListList.size()>=1){
-//                for (KnowledgeLog knowledgeLog : knowledgeLogListList) {
-//                    String name = knowledgeLog.getName().concat(".").concat(knowledgeLog.getFormat());
-//                    String sha256 = knowledgeLog.getSha256();
-//                    updateMap.put(name,sha256);
-//                }
-//            }
-//        }
+
 
         configService.addListener(NACOS_DATA_ID, NACOS_GROUP, new Listener() {
             @Override
@@ -112,13 +121,14 @@ public class SingleHttpSource extends RichHttpSourceFunction<Map<String, byte[]>
                                         knowledgeJson.getStr("format"));
                                 String sha256 = knowledgeJson.getStr("sha256");
                                 String filePath = knowledgeJson.getStr("path");
-                                if (!sha256.equals(updateMap.get(fileName))) {
-                                    updateMap.put(fileName, sha256);
-                                    updateKnowledge(fileName, filePath);
+                                if (!sha256.equals(knowledgeMetaCache.get(fileName))) {
+                                    knowledgeMetaCache.put(fileName, sha256);
+                                    updateKnowledge(fileName, filePath,sha256);
                                 }
-
                             }
-                            ctx.collect(knowledgeFileCache);
+                            if (isSending){
+                                ctx.collect(knowledgeFileCache);
+                            }
                         }
                     }
 
@@ -137,78 +147,105 @@ public class SingleHttpSource extends RichHttpSourceFunction<Map<String, byte[]>
 
     }
 
-//    private CustomFile loadKnowledge(String fileName, String filePath) {
+    private void loadKnowledge(ArrayList<Object> metaList) {
 //        InputStream inputStream = null;
-//        FileOutputStream outputStream = null;
-//        CustomFile customFile = new CustomFile();
-//        try {
-//            customFile.setFileName(fileName);
-//            Header header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
-//            HttpClientUtils2 httpClientUtils = new HttpClientUtils2();
-//            inputStream = httpClientUtils.httpGetInputStream(filePath, 3000, header);
-//            FileUtil.mkdir(CommonConfig.DOWNLOAD_PATH);
-//            File file = new File(CommonConfig.DOWNLOAD_PATH.concat(File.separator).concat(fileName));
-//            outputStream = new FileOutputStream(file);
-//            byte[] bytes = IOUtils.toByteArray(inputStream);
-//            customFile.setContent(bytes);
-//            inputStream = new ByteArrayInputStream(customFile.getContent());
-//            IoUtil.copy(inputStream, outputStream);
-//
-//        } catch (IOException ioException) {
-//            ioException.printStackTrace();
-//        } finally {
-//            IOUtils.closeQuietly(inputStream);
-//            IOUtils.closeQuietly(outputStream);
-//        }
-//        return customFile;
-//    }
-private void loadKnowledge(ArrayList<Object> metaList) {
-    InputStream inputStream = null;
-    try {
-        if (metaList.size() >= 1) {
-            for (Object metadata : metaList) {
-                JSONObject knowledgeJson = new JSONObject(metadata, false, true);
-                String fileName = Joiner.on(CommonConfig.LOCATION_SEPARATOR).useForNull("").join(knowledgeJson.getStr("name"),
-                        knowledgeJson.getStr("format"));
-                String sha256 = knowledgeJson.getStr("sha256");
-                String filePath = knowledgeJson.getStr("path");
-                Header header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
-                HttpClientUtils2 httpClientUtils = new HttpClientUtils2();
-                inputStream = httpClientUtils.httpGetInputStream(filePath, 3000, header);
-                updateMap.put(fileName, sha256);
-                knowledgeFileCache.put(fileName, IOUtils.toByteArray(inputStream));
-            }
-        }
-    } catch (IOException ioException) {
-        ioException.printStackTrace();
-    } finally {
-        IOUtils.closeQuietly(inputStream);
-    }
-}
-
-
-    private void updateKnowledge(String fileName, String filePath) {
-        InputStream inputStream = null;
-        FileOutputStream outputStream = null;
         try {
-            Header header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
-            HttpClientUtils2 httpClientUtils = new HttpClientUtils2();
-            inputStream = httpClientUtils.httpGetInputStream(filePath, 3000, header);
-            FileUtil.mkdir(CommonConfig.DOWNLOAD_PATH);
-            File file = new File(CommonConfig.DOWNLOAD_PATH.concat(File.separator).concat(fileName));
-            outputStream = new FileOutputStream(file);
-            byte[] bytes = IOUtils.toByteArray(inputStream);
-            knowledgeFileCache.put(fileName, bytes);
-            inputStream=new ByteArrayInputStream(bytes);
-            IoUtil.copy(inputStream, outputStream);
+            if (metaList.size() >= 1) {
+                for (Object metadata : metaList) {
+                    JSONObject knowledgeJson = new JSONObject(metadata, false, true);
+                    String fileName = Joiner.on(CommonConfig.LOCATION_SEPARATOR).useForNull("").join(knowledgeJson.getStr("name"),
+                            knowledgeJson.getStr("format"));
+                    String sha256 = knowledgeJson.getStr("sha256");
+                    String filePath = knowledgeJson.getStr("path");
+                    byte[] localFileByte = getLocalFile(fileName);
+                    String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
+                    if (sha256.equals(localFileSha256Hex)){
+                        logger.info("本地文件{}的sha256为:{} ,Nacos内记录为:{} ,sha256相等", fileName, localFileSha256Hex, sha256);
+                        knowledgeMetaCache.put(fileName, sha256);
+                        knowledgeFileCache.put(fileName, localFileByte);
+                    }else {
+                        logger.info("本地文件{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等，更新本地文件及缓存", fileName, localFileSha256Hex, sha256);
+                        updateKnowledge(fileName,filePath,sha256);
+                    }
+                }
+            }
+        } catch (RuntimeException exception) {
+            exception.printStackTrace();
+        }
+//        finally {
+//            IOUtils.closeQuietly(inputStream);
+//        }
+    }
+
+
+    private void updateKnowledge(String fileName, String filePath,String sha256) {
+        InputStream inputStream = null;
+//        FileOutputStream outputStream = null;
+        int retryNum = 0;
+        try {
+            while (retryNum < TRY_TIMES){
+                inputStream = httpClientUtils.httpGetInputStream(filePath, 3000, header);
+                if (inputStream !=null){
+                    byte[] downloadBytes = IOUtils.toByteArray(inputStream);
+                    if (downloadBytes.length>0){
+                        String downloadFileSha256Hex = DigestUtil.sha256Hex(downloadBytes);
+                        if (sha256.equals(downloadFileSha256Hex)){
+                            logger.info("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256相等", fileName, sha256);
+                            knowledgeMetaCache.put(fileName, sha256);
+                            knowledgeFileCache.put(fileName, downloadBytes);
+                            updateLocalFile(fileName);
+                            retryNum = TRY_TIMES;
+                            isSending = true;
+                        }else {
+                            logger.error("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等 开始第{}次重试下载文件", fileName, downloadFileSha256Hex, sha256, retryNum);
+                            retryNum++;
+                        }
+
+                    }
+                }
+            }
+
         } catch (IOException ioException) {
             ioException.printStackTrace();
         } finally {
             IOUtils.closeQuietly(inputStream);
-            IOUtils.closeQuietly(outputStream);
         }
     }
 
+
+
+    private void updateLocalFile(String fileName) {
+//        InputStream inputStream = null;
+        FileOutputStream outputStream = null;
+        try {
+            FileUtil.mkdir(CommonConfig.DOWNLOAD_PATH);
+            File file = new File(CommonConfig.DOWNLOAD_PATH.concat(File.separator).concat(fileName));
+            outputStream = new FileOutputStream(file);
+            IoUtil.copy(new ByteArrayInputStream(knowledgeFileCache.get(fileName)), outputStream);
+        } catch (IOException ioe) {
+            logger.error("更新本地文件{}时发生IO异常,异常信息为:", fileName, ioe.getMessage());
+            ioe.printStackTrace();
+        } catch (RuntimeException e) {
+            logger.error("更新本地文件{}时发生异常,异常信息为:", fileName, e.getMessage());
+            e.printStackTrace();
+        } finally {
+            IOUtils.closeQuietly(outputStream);
+        }
+
+    }
+
+    private static byte[] getLocalFile(String name) {
+        byte[] fileBytes = null;
+        try {
+            fileBytes=new FileReader(CommonConfig.DOWNLOAD_PATH + name).readBytes();
+        } catch (RuntimeException e) {
+            logger.error("IpLookupUtils download MMDB files error, message is:" + e.getMessage());
+            e.printStackTrace();
+        }
+        return fileBytes;
+    }
+
+
     @Override
     public void cancel() {
         this.isRunning = false;

src/main/java/com/zdjizhi/utils/DistributedLock.java

@@ -1,9 +1,9 @@
 package com.zdjizhi.utils;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import org.apache.zookeeper.*;
 import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -17,7 +17,8 @@ import java.util.concurrent.locks.Lock;
 
 
 public class DistributedLock implements Lock, Watcher {
-    private static final Logger logger = LoggerFactory.getLogger(DistributedLock.class);
+//    private static final Logger logger = LoggerFactory.getLogger(DistributedLock.class);
+    private static final Log logger = LogFactory.get();
 
     private ZooKeeper zk = null;
     /**

src/main/java/com/zdjizhi/utils/HdfsUtils.java

@@ -59,7 +59,7 @@ public class HdfsUtils {
     public static void uploadFileByBytes(String filePath,byte[] bytes) throws IOException {
         try (FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path(filePath), true)) {
             fsDataOutputStream.write(bytes);
-            fsDataOutputStream.flush();
+//            fsDataOutputStream.flush();
         } catch (RuntimeException e) {
             logger.error("Uploading files to the HDFS is abnormal. Message is :" + e.getMessage());
         } catch (IOException e) {

src/main/java/com/zdjizhi/utils/HttpClientUtils.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.utils;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import org.apache.http.*;
 import org.apache.http.client.ClientProtocolException;
@@ -18,8 +20,6 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.message.BasicHeaderElementIterator;
 import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLHandshakeException;
@@ -38,7 +38,8 @@ public class HttpClientUtils {
     /** 全局连接池对象 */
     private static final PoolingHttpClientConnectionManager CONN_MANAGER = new PoolingHttpClientConnectionManager();
 
-    private static Logger logger = LoggerFactory.getLogger(HttpClientUtils.class);
+//    private static Logger logger = LoggerFactory.getLogger(HttpClientUtils.class);
+    private static final Log logger = LogFactory.get();
     public static final String ERROR_MESSAGE = "-1";
 
     /*

src/main/java/com/zdjizhi/utils/NacosUtils.java

@@ -1,11 +1,11 @@
 package com.zdjizhi.utils;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.alibaba.nacos.api.NacosFactory;
 import com.alibaba.nacos.api.PropertyKeyConst;
 import com.alibaba.nacos.api.config.ConfigService;
 import com.alibaba.nacos.api.config.listener.Listener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -13,7 +13,8 @@ import java.util.Properties;
 import java.util.concurrent.Executor;
 
 public class NacosUtils {
-    private static final Logger logger = LoggerFactory.getLogger(NacosUtils.class);
+//    private static final Logger logger = LoggerFactory.getLogger(NacosUtils.class);
+    private static final Log logger = LogFactory.get();
     private static Properties nacosProperties = new Properties();
     private static Properties commonProperties = new Properties();
 

src/main/java/com/zdjizhi/utils/SnowflakeId.java

@@ -1,11 +1,12 @@
 package com.zdjizhi.utils;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public class SnowflakeId {
-    private static final Logger logger = LoggerFactory.getLogger(SnowflakeId.class);
+//    private static final Logger logger = LoggerFactory.getLogger(SnowflakeId.class);
+    private static final Log logger = LogFactory.get();
 
     /**
      * 共64位 第一位为符号位 默认0

src/main/java/com/zdjizhi/utils/ZookeeperUtils.java

@@ -1,11 +1,11 @@
 package com.zdjizhi.utils;
 
 import cn.hutool.core.util.StrUtil;
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import org.apache.zookeeper.*;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
@@ -13,7 +13,8 @@ import java.util.concurrent.CountDownLatch;
 
 
 public class ZookeeperUtils implements Watcher {
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperUtils.class);
+//    private static final Logger logger = LoggerFactory.getLogger(ZookeeperUtils.class);
+    private static final Log logger = LogFactory.get();
 
     private ZooKeeper zookeeper;
 

src/main/resources/common.properties

@@ -5,7 +5,7 @@ stream.execution.environment.parallelism=1
 stream.execution.job.name=DOS-DETECTION-APPLICATION
 
 #输入kafka并行度大小
-kafka.input.parallelism=1
+kafka.input.parallelism=3
 
 #输入kafka topic名
 kafka.input.topic.name=DOS-SKETCH-RECORD
@@ -15,18 +15,18 @@ kafka.input.topic.name=DOS-SKETCH-RECORD
 kafka.input.bootstrap.servers=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
 
 #读取kafka group id
-kafka.input.group.id=2112080949
+kafka.input.group.id=dos-detection-job-221125-1
 #kafka.input.group.id=dos-detection-job-210813-1
 
 #发送kafka metrics并行度大小
-kafka.output.metric.parallelism=1
+kafka.output.metric.parallelism=3
 
 #发送kafka metrics topic名
 #kafka.output.metric.topic.name=TRAFFIC-TOP-DESTINATION-IP-METRICS
 kafka.output.metric.topic.name=test
 
 #发送kafka event并行度大小
-kafka.output.event.parallelism=1
+kafka.output.event.parallelism=3
 
 #发送kafka event topic名
 #kafka.output.event.topic.name=DOS-EVENT
@@ -52,7 +52,7 @@ hbase.baseline.table.name=dos:ddos_traffic_baselines
 hbase.baseline.total.num=1000000
 
 #baseline ttl，单位：天
-hbase.baseline.ttl=30
+hbase.baseline.ttl=10
 
 #设置聚合并行度，2个key
 flink.first.agg.parallelism=1
@@ -61,10 +61,10 @@ flink.first.agg.parallelism=1
 flink.detection.map.parallelism=1
 
 #watermark延迟
-flink.watermark.max.orderness=10
+flink.watermark.max.orderness=300
 
 #计算窗口大小，默认600s
-flink.window.max.time=10
+flink.window.max.time=600
 
 #dos event结果中distinct source IP限制
 source.ip.list.limit=10000
@@ -83,9 +83,6 @@ ip.mmdb.path=D:\\data\\dat\\bak\\
 bifang.server.uri=http://192.168.44.72:80
 #bifang.server.uri=http://192.168.44.3:80
 
-#访问bifang只读权限token，bifang内置，无需修改
-bifang.server.token=ed04b942-7df4-4e3d-b9a9-a881ca98a867
-
 #加密密码路径信息
 bifang.server.encryptpwd.path=/v1/user/encryptpwd
 
@@ -123,23 +120,11 @@ baseline.threshold.schedule.days=1
 #kafka用户认证配置参数
 sasl.jaas.config.user=admin
 #sasl.jaas.config.password=galaxy2019
-#sasl.jaas.config.password=ENC(6MleDyA3Z73HSaXiKsDJ2k7Ys8YWLhEJ)
 sasl.jaas.config.password=6MleDyA3Z73HSaXiKsDJ2k7Ys8YWLhEJ
 
 #是否开启kafka用户认证配置，1：是；0：否
 sasl.jaas.config.flag=1
 
-#nacos配置
-#nacos.server.addr=192.168.44.12:8848
-#nacos.namespace=public
-#nacos.username=nacos
-#nacos.password=nacos
-#nacos.data.id=knowledge_base.json
-#nacos.group=DEFAULT_GROUP
-#nacos.read.timeout=5000
-
-
-
 ############################## Nacos 配置 ######################################
 nacos.server.addr=192.168.44.12:8848
 nacos.username=nacos
@@ -156,30 +141,6 @@ nacos.static.namespace=test
 nacos.static.data.id=dos_detection.properties
 nacos.static.group=Galaxy
 
-############################## HTTP 配置 ######################################
-#http请求相关参数
-#最大连接数
-#http.pool.max.connection=400
-#
-##单路由最大连接数
-#http.pool.max.per.route=80
-#
-##向服务端请求超时时间设置(单位:毫秒)
-#http.pool.request.timeout=60000
-#
-##向服务端连接超时时间设置(单位:毫秒)
-#http.pool.connect.timeout=60000
-#
-##服务端响应超时时间设置(单位:毫秒)
-#http.pool.response.timeout=60000
-
-
-#server.uri=http://192.168.44.12:9098
-#server.path=/hos/knowledge_base_hos_bucket
-
-
-
-
 ############################## hos Token 配置 ######################################
 hos.token=c21f969b5f03d33d43e04f8f136e7682
 
@@ -189,8 +150,8 @@ cluster.or.single=CLUSTER
 
 ############################## 集群模式配置文件路径 配置 ######################################
 hdfs.path=/test/TEST/
-hdfs.uri.nn1=hdfs://192.168.40.151:9000
-hdfs.uri.nn2=hdfs://192.168.40.152:9000
+hdfs.uri.nn1=192.168.40.151:9000
+hdfs.uri.nn2=192.168.40.152:9000
 hdfs.user=dos
 
 ############################## 单机模式配置文件下载路径 配置 ######################################

src/main/resources/log4j.properties

@@ -0,0 +1,23 @@
+#Log4j
+log4j.rootLogger=info,console,file
+# 控制台日志设置
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=info
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] [Thread\:%t] %l %x - <%m>%n
+
+# 文件日志设置
+log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.file.Threshold=info
+log4j.appender.file.encoding=UTF-8
+log4j.appender.file.Append=true
+#路径请用相对路径，做好相关测试输出到应用目下
+log4j.appender.file.file=${nis.root}/log/flink-dos-detection.log
+log4j.appender.file.DatePattern='.'yyyy-MM-dd
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+#log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss} %X{ip} [%t] %5p %c{1} %m%n
+log4j.appender.file.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] %X{ip} [Thread\:%t]  %l %x - %m%n
+#MyBatis 配置，com.nis.web.dao是mybatis接口所在包
+log4j.logger.com.nis.web.dao=debug
+#bonecp数据源配置
+log4j.category.com.jolbox=debug,console