1.适配IP定位库v4/v6合并后的加载逻辑（GAL-436）。

2.适配bifang23.11接口（TSG-17675）。
3.移除广播流。
4.修复静态阈值判断部分的BUG。

pom.xml

@@ -6,9 +6,14 @@
 
     <groupId>com.zdjizhi</groupId>
     <artifactId>flink-dos-detection</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>23.11</version>
+
+    <name>flink-dos-detection</name>
+    <url>http://www.example.com</url>
+
 
     <properties>
+        <galaxy.tools.version>1.2.1</galaxy.tools.version>
         <flink.version>1.13.1</flink.version>
         <hive.version>2.1.1</hive.version>
         <hadoop.version>2.7.1</hadoop.version>
@@ -152,7 +157,6 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-kafka_2.12</artifactId>
             <version>${flink.version}</version>
-            <!--<scope>provided</scope>-->
         </dependency>
 
         <!-- CLI dependencies -->
@@ -160,7 +164,7 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-clients_2.12</artifactId>
             <version>${flink.version}</version>
-            <!--<scope>provided</scope>-->
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -203,7 +207,7 @@
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-client</artifactId>
             <version>2.2.3</version>
-            <!--<scope>provided</scope>-->
+            <scope>provided</scope>
             <exclusions>
                 <exclusion>
                     <artifactId>slf4j-log4j12</artifactId>
@@ -258,26 +262,6 @@
             <version>5.3.3</version>
         </dependency>
 
-        <dependency>
-            <groupId>com.zdjizhi</groupId>
-            <artifactId>galaxy</artifactId>
-            <version>1.1.3</version>
-            <exclusions>
-                <exclusion>
-                    <artifactId>slf4j-log4j12</artifactId>
-                    <groupId>org.slf4j</groupId>
-                </exclusion>
-                <exclusion>
-                    <artifactId>log4j-over-slf4j</artifactId>
-                    <groupId>org.slf4j</groupId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.guava</groupId>
-                    <artifactId>guava</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
 
         <dependency>
             <groupId>com.alibaba.fastjson2</groupId>
@@ -354,6 +338,12 @@
         </dependency>
 
 
+
+        <dependency>
+            <groupId>com.geedgenetworks</groupId>
+            <artifactId>galaxy</artifactId>
+            <version>${galaxy.tools.version}</version>
+        </dependency>
     </dependencies>
 
 

src/main/java/com/zdjizhi/common/CustomFile.java

@@ -1,26 +0,0 @@
-package com.zdjizhi.common;
-
-import java.io.Serializable;
-
-public class CustomFile implements Serializable {
-
-    String fileName;
-
-    byte[] content;
-
-    public String getFileName() {
-        return fileName;
-    }
-
-    public void setFileName(String fileName) {
-        this.fileName = fileName;
-    }
-
-    public byte[] getContent() {
-        return content;
-    }
-
-    public void setContent(byte[] content) {
-        this.content = content;
-    }
-}

src/main/java/com/zdjizhi/common/DosDetectionThreshold.java

@@ -9,110 +9,110 @@ import java.util.Objects;
  * @author wlh
  */
 public class DosDetectionThreshold implements Serializable {
-    private long profileId;
+    private long profile_id;
-    private String attackType;
+    private String attack_type;
-    private ArrayList<String> serverIpList;
+    private ArrayList<String> server_ip_list;
-    private String serverIpAddr;
+    private String server_ip_addr;
-    private long packetsPerSec;
+    private long packets_per_sec;
-    private long bitsPerSec;
+    private long bits_per_sec;
-    private long sessionsPerSec;
+    private long sessions_per_sec;
-    private int isValid;
+    private int is_valid;
-    private int vsysId;
+    private int vsys_id;
-    private Integer[] superiorIds;
+    private Integer[] superior_ids;
 
-    public long getProfileId() {
+    public long getProfile_id() {
-        return profileId;
+        return profile_id;
     }
 
-    public void setProfileId(long profileId) {
+    public void setProfile_id(long profile_id) {
-        this.profileId = profileId;
+        this.profile_id = profile_id;
     }
 
-    public String getAttackType() {
+    public String getAttack_type() {
-        return attackType;
+        return attack_type;
     }
 
-    public void setAttackType(String attackType) {
+    public void setAttack_type(String attack_type) {
-        this.attackType = attackType;
+        this.attack_type = attack_type;
     }
 
-    public ArrayList<String> getServerIpList() {
+    public ArrayList<String> getServer_ip_list() {
-        return serverIpList;
+        return server_ip_list;
     }
 
-    public void setServerIpList(ArrayList<String> serverIpList) {
+    public void setServer_ip_list(ArrayList<String> server_ip_list) {
-        this.serverIpList = serverIpList;
+        this.server_ip_list = server_ip_list;
     }
 
-    public String getServerIpAddr() {
+    public String getServer_ip_addr() {
-        return serverIpAddr;
+        return server_ip_addr;
     }
 
-    public void setServerIpAddr(String serverIpAddr) {
+    public void setServer_ip_addr(String server_ip_addr) {
-        this.serverIpAddr = serverIpAddr;
+        this.server_ip_addr = server_ip_addr;
     }
 
-    public long getPacketsPerSec() {
+    public long getPackets_per_sec() {
-        return packetsPerSec;
+        return packets_per_sec;
     }
 
-    public void setPacketsPerSec(long packetsPerSec) {
+    public void setPackets_per_sec(long packets_per_sec) {
-        this.packetsPerSec = packetsPerSec;
+        this.packets_per_sec = packets_per_sec;
     }
 
-    public long getBitsPerSec() {
+    public long getBits_per_sec() {
-        return bitsPerSec;
+        return bits_per_sec;
     }
 
-    public void setBitsPerSec(long bitsPerSec) {
+    public void setBits_per_sec(long bits_per_sec) {
-        this.bitsPerSec = bitsPerSec;
+        this.bits_per_sec = bits_per_sec;
     }
 
-    public long getSessionsPerSec() {
+    public long getSessions_per_sec() {
-        return sessionsPerSec;
+        return sessions_per_sec;
     }
 
-    public void setSessionsPerSec(long sessionsPerSec) {
+    public void setSessions_per_sec(long sessions_per_sec) {
-        this.sessionsPerSec = sessionsPerSec;
+        this.sessions_per_sec = sessions_per_sec;
     }
 
-    public int getIsValid() {
+    public int getIs_valid() {
-        return isValid;
+        return is_valid;
     }
 
-    public void setIsValid(int isValid) {
+    public void setIs_valid(int is_valid) {
-        this.isValid = isValid;
+        this.is_valid = is_valid;
     }
 
-    public int getVsysId() {
+    public int getVsys_id() {
-        return vsysId;
+        return vsys_id;
     }
 
-    public void setVsysId(int vsysId) {
+    public void setVsys_id(int vsys_id) {
-        this.vsysId = vsysId;
+        this.vsys_id = vsys_id;
     }
 
-    public Integer[] getSuperiorIds() {
+    public Integer[] getSuperior_ids() {
-        return superiorIds;
+        return superior_ids;
     }
 
-    public void setSuperiorIds(Integer[] superiorIds) {
+    public void setSuperior_ids(Integer[] superior_ids) {
-        this.superiorIds = superiorIds;
+        this.superior_ids = superior_ids;
     }
 
     @Override
     public String toString() {
         return "DosDetectionThreshold{" +
-                "profileId='" + profileId + '\'' +
+                "profile_id=" + profile_id +
-                ", attackType='" + attackType + '\'' +
+                ", attack_type='" + attack_type + '\'' +
-                ", serverIpList=" + serverIpList +
+                ", server_ip_list=" + server_ip_list +
-                ", serverIpAddr='" + serverIpAddr + '\'' +
+                ", server_ip_addr='" + server_ip_addr + '\'' +
-                ", packetsPerSec=" + packetsPerSec +
+                ", packets_per_sec=" + packets_per_sec +
-                ", bitsPerSec=" + bitsPerSec +
+                ", bits_per_sec=" + bits_per_sec +
-                ", sessionsPerSec=" + sessionsPerSec +
+                ", sessions_per_sec=" + sessions_per_sec +
-                ", isValid=" + isValid +
+                ", is_valid=" + is_valid +
-                ", vsysId=" + vsysId +
+                ", vsys_id=" + vsys_id +
-                ", superiorIds=" + Arrays.toString(superiorIds) +
+                ", superior_ids=" + Arrays.toString(superior_ids) +
                 '}';
     }
 }

src/main/java/com/zdjizhi/common/DosVsysId.java

@@ -4,7 +4,7 @@ import java.util.Arrays;
 
 public class DosVsysId {
     private Integer id;
-    private Integer[] superiorIds;
+    private Integer[] superior_ids;
 
     public Integer getId() {
         return id;
@@ -14,19 +14,19 @@ public class DosVsysId {
         this.id = id;
     }
 
-    public Integer[] getSuperiorIds() {
+    public Integer[] getSuperior_ids() {
-        return superiorIds;
+        return superior_ids;
     }
 
-    public void setSuperiorIds(Integer[] superiorIds) {
+    public void setSuperior_ids(Integer[] superior_ids) {
-        this.superiorIds = superiorIds;
+        this.superior_ids = superior_ids;
     }
 
     @Override
     public String toString() {
         return "DosVsysId{" +
                 "id=" + id +
-                ", superiorIds=" + Arrays.toString(superiorIds) +
+                ", superior_ids=" + Arrays.toString(superior_ids) +
                 '}';
     }
 }

src/main/java/com/zdjizhi/common/FlowWriteConfig.java

@@ -7,12 +7,11 @@ import org.jasypt.encryption.pbe.StandardPBEStringEncryptor;
  * @author wlh
  * @date 2021/1/6
  */
-public class CommonConfig {
+public class FlowWriteConfig {
 
     /**
      * 定位库默认分隔符
      */
-    public static final String LOCATION_SEPARATOR = ".";
 
     private static StandardPBEStringEncryptor encryptor = new StandardPBEStringEncryptor();
 
@@ -51,7 +50,6 @@ public class CommonConfig {
     public static final int DESTINATION_IP_PARTITION_NUM = CommonConfigurations.getIntProperty("destination.ip.partition.num");
     public static final int DATA_CENTER_ID_NUM = CommonConfigurations.getIntProperty("data.center.id.num");
 
-    public static final String IP_MMDB_PATH = CommonConfigurations.getStringProperty("ip.mmdb.path");
 
     public static final String BIFANG_SERVER_URI = CommonConfigurations.getStringProperty("bifang.server.uri");
     public static final String BIFANG_SERVER_ENCRYPTPWD_PATH = CommonConfigurations.getStringProperty("bifang.server.encryptpwd.path");
@@ -74,37 +72,20 @@ public class CommonConfig {
 
     public static final int SASL_JAAS_CONFIG_FLAG = CommonConfigurations.getIntProperty("sasl.jaas.config.flag");
 
-    public static final String NACOS_SERVER_ADDR = CommonConfigurations.getStringProperty("nacos.server.addr");
+    public static final String NACOS_SERVER = CommonConfigurations.getStringProperty("nacos.server.addr");
     public static final String NACOS_USERNAME = CommonConfigurations.getStringProperty("nacos.username");
-    public static final String NACOS_PASSWORD = CommonConfigurations.getStringProperty("nacos.password");
+    public static final String NACOS_PIN = CommonConfigurations.getStringProperty("nacos.password");
-    public static final String NACOS_NAMESPACE = CommonConfigurations.getStringProperty("nacos.namespace");
+    public static final String NACOS_PUBLIC_NAMESPACE = CommonConfigurations.getStringProperty("nacos.namespace");
-    public static final String NACOS_DATA_ID = CommonConfigurations.getStringProperty("nacos.data.id");
+    public static final String NACOS_KNOWLEDGEBASE_DATA_ID = CommonConfigurations.getStringProperty("nacos.data.id");
-    public static final String NACOS_GROUP = CommonConfigurations.getStringProperty("nacos.group");
+    public static final String NACOS_PUBLIC_GROUP = CommonConfigurations.getStringProperty("nacos.group");
-    public static final int NACOS_READ_TIMEOUT = CommonConfigurations.getIntProperty("nacos.read.timeout");
+    public static final Integer NACOS_CONNECTION_TIMEOUT = CommonConfigurations.getIntProperty("nacos.connection.timeout");
 
 
-    public static final String HOS_TOKEN = CommonConfigurations.getStringProperty("hos.token");
 
-    public static final String CLUSTER_OR_SINGLE = CommonConfigurations.getStringProperty("cluster.or.single");
+    public static final String NACOS_DOS_NAMESPACE = CommonConfigurations.getStringProperty("nacos.dos.namespace");
+    public static final String NACOS_DOS_DATA_ID = CommonConfigurations.getStringProperty("nacos.dos.data.id");
+    public static final String NACOS_DOS_GROUP = CommonConfigurations.getStringProperty("nacos.dos.group");
 
-    public static final String HDFS_URI_NS1 = CommonConfigurations.getStringProperty("hdfs.uri.nn1");
+    public static final Integer HTTP_SOCKET_TIMEOUT = CommonConfigurations.getIntProperty("http.socket.timeout");
-    public static final String HDFS_URI_NS2 = CommonConfigurations.getStringProperty("hdfs.uri.nn2");
-    public static final String HDFS_PATH = CommonConfigurations.getStringProperty("hdfs.path");
-    public static final String HDFS_USER = CommonConfigurations.getStringProperty("hdfs.user");
 
-    public static final String DOWNLOAD_PATH = CommonConfigurations.getStringProperty("download.path");
-
-    public static void main(String[] args) {
-        StandardPBEStringEncryptor encryptor = new StandardPBEStringEncryptor();
-        // 配置加密解密的密码/salt值
-        encryptor.setPassword("galaxy");
-        // 对"raw_password"进行加密：S5kR+Y7CI8k7MaecZpde25yK8NKUnd6p
-//        String password = "galaxy2019";
-        String password = "nacos";
-        String encPwd = encryptor.encrypt(password);
-        System.out.println(encPwd);
-        // 再进行解密：raw_password
-        String rawPwd = encryptor.decrypt(encPwd);
-        System.out.println(rawPwd);
-    }
 }

src/main/java/com/zdjizhi/common/pojo/KnowlegeBaseMeta.java

@@ -1,14 +1,24 @@
-package com.zdjizhi.common;
+package com.zdjizhi.common.pojo;
 
-public class KnowledgeLog {
+import java.io.Serializable;
-    public   String id;
+
-    public   String name;
+/**
-    public   String path;
+ *
-    public   Long size;
+ */
-    public   String format;
+public class KnowlegeBaseMeta implements Serializable {
-    public   String sha256;
+    private String id;
-    public   String version;
+    private String name;
-    public   String updateTime;
+    private String sha256;
+    private String format;
+    private String path;
+
+    public KnowlegeBaseMeta(String id, String name, String sha256, String format, String path) {
+        this.id = id;
+        this.name = name;
+        this.sha256 = sha256;
+        this.format = format;
+        this.path = path;
+    }
 
     public String getId() {
         return id;
@@ -26,20 +36,12 @@ public class KnowledgeLog {
         this.name = name;
     }
 
-    public String getPath() {
+    public String getSha256() {
-        return path;
+        return sha256;
     }
 
-    public void setPath(String path) {
+    public void setSha256(String sha256) {
-        this.path = path;
+        this.sha256 = sha256;
-    }
-
-    public Long getSize() {
-        return size;
-    }
-
-    public void setSize(Long size) {
-        this.size = size;
     }
 
     public String getFormat() {
@@ -50,42 +52,23 @@ public class KnowledgeLog {
         this.format = format;
     }
 
-    public String getSha256() {
+    public String getPath() {
-        return sha256;
+        return path;
     }
 
-    public void setSha256(String sha256) {
+    public void setPath(String path) {
-        this.sha256 = sha256;
+        this.path = path;
     }
 
-    public String getVersion() {
-        return version;
-    }
-
-    public void setVersion(String version) {
-        this.version = version;
-    }
-
-    public String getUpdateTime() {
-        return updateTime;
-    }
-
-    public void setUpdateTime(String updateTime) {
-        this.updateTime = updateTime;
-    }
-
-
     @Override
     public String toString() {
-        return "KnowledgeLog{" +
+        return "KnowlegeBaseMeta{" +
                 "id='" + id + '\'' +
                 ", name='" + name + '\'' +
-                ", path='" + path + '\'' +
-                ", size=" + size +
-                ", format='" + format + '\'' +
                 ", sha256='" + sha256 + '\'' +
-                ", version='" + version + '\'' +
+                ", format='" + format + '\'' +
-                ", updateTime='" + updateTime + '\'' +
+                ", path='" + path + '\'' +
                 '}';
     }
 }
+

src/main/java/com/zdjizhi/etl/DosDetection.java

@@ -1,17 +1,19 @@
 package com.zdjizhi.etl;
 
-import cn.hutool.core.math.MathUtil;
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
+import com.geedgenetworks.utils.DateUtils;
+import com.geedgenetworks.utils.StringUtil;
 import com.zdjizhi.common.*;
 import com.zdjizhi.utils.*;
+import com.zdjizhi.utils.connections.nacos.NacosUtils;
 import inet.ipaddr.IPAddress;
 import inet.ipaddr.IPAddressString;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap;
-import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
 import org.apache.flink.util.Collector;
 
 import java.math.BigDecimal;
@@ -24,12 +26,12 @@ import java.util.concurrent.TimeUnit;
 /**
  * @author wlh
  */
-public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<String, String>, DosEventLog> {
+public class DosDetection extends ProcessFunction<DosSketchLog, DosEventLog> {
 
     private static final Log logger = LogFactory.get();
     private static Map<String, Map<String, DosBaselineThreshold>> baselineMap = new HashMap<>();
     private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance();
-    private HashMap<Integer,HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> thresholdRangeMap;
+    private HashMap<Integer, HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> thresholdRangeMap;
 
     private final static int BASELINE_SIZE = 144;
     private final static int STATIC_CONDITION_TYPE = 1;
@@ -42,22 +44,20 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
 
     private final static int OTHER_BASELINE_TYPE = 3;
 
+
+
+
     @Override
     public void open(Configuration parameters) {
         ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(2,
                 new BasicThreadFactory.Builder().namingPattern("Dos-Detection-%d").daemon(true).build());
         try {
-
             super.open(parameters);
-            logger.info("begin init");
-            IpUtils.loadIpLook();
-            logger.info("init over");
-
             executorService.scheduleAtFixedRate(() -> thresholdRangeMap = ParseStaticThreshold.createStaticThreshold(), 0,
-                    CommonConfig.STATIC_THRESHOLD_SCHEDULE_MINUTES, TimeUnit.MINUTES);
+                    FlowWriteConfig.STATIC_THRESHOLD_SCHEDULE_MINUTES, TimeUnit.MINUTES);
 
             executorService.scheduleAtFixedRate(() -> baselineMap = ParseBaselineThreshold.readFromHbase(), 0,
-                    CommonConfig.BASELINE_THRESHOLD_SCHEDULE_DAYS, TimeUnit.DAYS);
+                    FlowWriteConfig.BASELINE_THRESHOLD_SCHEDULE_DAYS, TimeUnit.DAYS);
         } catch (Exception e) {
             logger.error("定时器任务执行失败", e);
         }
@@ -65,7 +65,7 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
     }
 
     @Override
-    public void processElement(DosSketchLog value, ReadOnlyContext ctx, Collector<DosEventLog> out) {
+    public void processElement(DosSketchLog value, Context ctx, Collector<DosEventLog> out) throws Exception {
         DosEventLog finalResult = null;
         try {
             String destinationIp = value.getDestination_ip();
@@ -75,13 +75,13 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
             IPAddress destinationIpAddress = new IPAddressString(destinationIp).getAddress();
 
             DosDetectionThreshold threshold = null;
-            if (thresholdRangeMap.containsKey(vsysId)){
+            if (thresholdRangeMap.containsKey(vsysId)) {
                 threshold = thresholdRangeMap.get(vsysId).getOrDefault(attackType, TreeRangeMap.create()).get(destinationIpAddress);
             }
 
             logger.debug("当前判断IP：{}, 类型: {}", key, attackType);
             if (threshold == null && baselineMap.containsKey(key)) {
-                finalResult = getDosEventLogByBaseline(value,key);
+                finalResult = getDosEventLogByBaseline(value, key);
             } else if (threshold == null && !baselineMap.containsKey(key)) {
                 finalResult = getDosEventLogBySensitivityThreshold(value);
             } else if (threshold != null) {
@@ -94,71 +94,62 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
             logger.error("判定失败\n {} \n{}", value, e);
         }
 
-        if (finalResult != null){
+        if (finalResult != null) {
             out.collect(finalResult);
         }
     }
 
-    @Override
-    public void processBroadcastElement(Map<String, String> value, Context ctx, Collector<DosEventLog> out) throws Exception {
-        if (!value.isEmpty()){
-            IpUtils.updateIpLook(value);
-        }
-    }
 
     private DosEventLog getDosEventLogBySensitivityThreshold(DosSketchLog value) {
         long sketchSessions = value.getSketch_sessions();
         Integer staticSensitivityThreshold = NacosUtils.getIntProperty("static.sensitivity.threshold");
         long diff = sketchSessions - staticSensitivityThreshold;
-        return getDosEventLog(value, staticSensitivityThreshold, diff,0, SENSITIVITY_CONDITION_TYPE, SESSIONS_TAG);
+        return getDosEventLog(value, staticSensitivityThreshold, diff, 0, SENSITIVITY_CONDITION_TYPE, SESSIONS_TAG);
     }
 
-    private DosEventLog getDosEventLogByBaseline(DosSketchLog value,String key) {
+    private DosEventLog getDosEventLogByBaseline(DosSketchLog value, String key) {
         String attackType = value.getAttack_type();
         long sketchSessions = value.getSketch_sessions();
         DosBaselineThreshold dosBaselineThreshold = baselineMap.get(key).get(attackType);
         Integer base = getBaseValue(dosBaselineThreshold, value);
         long diff = sketchSessions - base;
-        return getDosEventLog(value, base, diff, 0,BASELINE_CONDITION_TYPE, SESSIONS_TAG);
+        return getDosEventLog(value, base, diff, 0, BASELINE_CONDITION_TYPE, SESSIONS_TAG);
     }
 
     private DosEventLog getDosEventLogByStaticThreshold(DosSketchLog value, DosDetectionThreshold threshold) throws CloneNotSupportedException {
-        long sessionBase = threshold.getSessionsPerSec();
+        long sessionBase = threshold.getSessions_per_sec();
-        long pktBase=threshold.getPacketsPerSec();
+        long pktBase = threshold.getPackets_per_sec();
-        long bitBase=threshold.getBitsPerSec();
+        long bitBase = threshold.getBits_per_sec();
 
         long diffSession = value.getSketch_sessions() - sessionBase;
         long diffPkt = value.getSketch_packets() - pktBase;
         long diffByte = value.getSketch_bytes() - bitBase;
 
-//        Double diffSessionPercent = getDiffPercent(diffSession, sessionBase)*100;
+        double diffSessionPercent = 0.0;
-//        Double diffPktPercent = getDiffPercent(diffPkt, pktBase)*100;
+        double diffPktPercent = 0.0;
-//        Double diffBitPercent = getDiffPercent(diffByte, bitBase)*100;
+        double diffBitPercent = 0.0;
-
+        //todo 代码Review发现该部分存在bug，23.11版本做修复，需测试。
-        double diffSessionPercent=0.0;
+        if (sessionBase > 0) {
-        double diffPktPercent=0.0;
+            diffSessionPercent = getDiffPercent(diffSession, sessionBase) * 100;
-        double diffBitPercent=0.0;
-        if (sessionBase != 0 && sessionBase > 0){
-            diffSessionPercent = getDiffPercent(diffSession, sessionBase)*100;
         }
-        else if (pktBase != 0 && pktBase > 0){
+        if (pktBase > 0) {
-            diffPktPercent = getDiffPercent(diffPkt, pktBase)*100;
+            diffPktPercent = getDiffPercent(diffPkt, pktBase) * 100;
         }
-        else if (bitBase != 0 && bitBase > 0){
+        if (bitBase > 0) {
-            diffBitPercent = getDiffPercent(diffByte, bitBase)*100;
+            diffBitPercent = getDiffPercent(diffByte, bitBase) * 100;
         }
 
         long profileId = 0;
-        DosEventLog result =null;
+        DosEventLog result = null;
 
-        if (diffSessionPercent >= diffPktPercent && diffSessionPercent >= diffBitPercent){
+        if (diffSessionPercent >= diffPktPercent && diffSessionPercent >= diffBitPercent) {
-            profileId = threshold.getProfileId();
+            profileId = threshold.getProfile_id();
-            result= getDosEventLog(value, sessionBase, diffSession, profileId, STATIC_CONDITION_TYPE, SESSIONS_TAG);
+            result = getDosEventLog(value, sessionBase, diffSession, profileId, STATIC_CONDITION_TYPE, SESSIONS_TAG);
-        }else if (diffPktPercent >= diffSessionPercent && diffPktPercent >= diffBitPercent){
+        } else if (diffPktPercent >= diffSessionPercent && diffPktPercent >= diffBitPercent) {
-            profileId = threshold.getProfileId();
+            profileId = threshold.getProfile_id();
-            result = getDosEventLog(value, pktBase, diffPkt,profileId, STATIC_CONDITION_TYPE, PACKETS_TAG);
+            result = getDosEventLog(value, pktBase, diffPkt, profileId, STATIC_CONDITION_TYPE, PACKETS_TAG);
-        }else if (diffBitPercent >= diffPktPercent && diffBitPercent >= diffSessionPercent){
+        } else if (diffBitPercent >= diffPktPercent && diffBitPercent >= diffSessionPercent) {
-            profileId = threshold.getProfileId();
+            profileId = threshold.getProfile_id();
             result = getDosEventLog(value, bitBase, diffByte, profileId, STATIC_CONDITION_TYPE, BITS_TAG);
         }
         return result;
@@ -175,15 +166,15 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
             if (severity != Severity.NORMAL) {
                 if (type == BASELINE_CONDITION_TYPE && percent < NacosUtils.getDoubleProperty("baseline.sensitivity.threshold")) {
                     logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过基线敏感阈值,日志详情\n{}", destinationIp, attackType, base, percent, value);
-                }else if ((type == BASELINE_CONDITION_TYPE || type == SENSITIVITY_CONDITION_TYPE) && value.getSketch_sessions() < staticSensitivityThreshold){
+                } else if ((type == BASELINE_CONDITION_TYPE || type == SENSITIVITY_CONDITION_TYPE) && value.getSketch_sessions() < staticSensitivityThreshold) {
-                    logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过静态敏感阈值,日志详情\n{}",destinationIp, attackType, base, percent, value);
+                    logger.debug("当前server IP:{},类型:{},基线值{}百分比{}未超过静态敏感阈值,日志详情\n{}", destinationIp, attackType, base, percent, value);
-                }else {
+                } else {
 //                    result = getResult(value, base, profileId, severity, percent+1, type, tag);
                     result = getResult(value, base, profileId, severity, percent, type, tag);
-                    if (type == SENSITIVITY_CONDITION_TYPE){
+                    if (type == SENSITIVITY_CONDITION_TYPE) {
                         result.setSeverity(Severity.MAJOR.severity);
                     }
-                    logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,基于{}:{}检测,日志详情\n {}", destinationIp,attackType,base,percent,type,tag,result);
+                    logger.info("检测到当前server IP {} 存在 {} 异常,超出基线{} {}倍,基于{}:{}检测,日志详情\n {}", destinationIp, attackType, base, percent, type, tag, result);
                 }
             } else {
                 logger.debug("当前server IP:{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value);
@@ -201,10 +192,10 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
         dosEventLog.setProfile_id(profileId);
         dosEventLog.setAttack_type(value.getAttack_type());
         dosEventLog.setSeverity(severity.severity);
-        dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent), base, value.getSketch_sessions(), type, tag,dosEventLog));
+        dosEventLog.setConditions(getConditions(PERCENT_INSTANCE.format(percent), base, value.getSketch_sessions(), type, tag, dosEventLog));
 //        dosEventLog.setConditions(getConditions(percent, base, value.getSketch_sessions(), type, tag,dosEventLog));
         dosEventLog.setDestination_ip(value.getDestination_ip());
-        dosEventLog.setDestination_country(IpUtils.ipLookup.countryLookup(value.getDestination_ip()));
+        dosEventLog.setDestination_country(IpLookupUtils.getCountryLookup(value.getDestination_ip()));
         String ipList = value.getSource_ip();
         dosEventLog.setSource_ip_list(ipList);
         dosEventLog.setSource_country_list(getSourceCountryList(ipList));
@@ -228,7 +219,7 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
                         logger.debug("获取到当前IP: {},类型: {} baseline值为0,替换为P95观测值{}", value.getDestination_ip(), value.getAttack_type(), defaultVaule);
                         base = defaultVaule;
                     }
-                    if (sessionRateBaselineType == OTHER_BASELINE_TYPE && base < NacosUtils.getIntProperty("static.sensitivity.threshold")){
+                    if (sessionRateBaselineType == OTHER_BASELINE_TYPE && base < NacosUtils.getIntProperty("static.sensitivity.threshold")) {
                         base = NacosUtils.getIntProperty("static.sensitivity.threshold");
                     }
                 }
@@ -239,24 +230,24 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
         return base;
     }
 
-    private String getConditions(String percent, long base, long sessions, int type, String tag,DosEventLog dosEventLog) {
+    private String getConditions(String percent, long base, long sessions, int type, String tag, DosEventLog dosEventLog) {
-        int condition =0;
+        int condition = 0;
-        if ("Minor".equals(dosEventLog.getSeverity())){
+        if ("Minor".equals(dosEventLog.getSeverity())) {
-            condition=50;
+            condition = 50;
-        }else if ("Warning".equals(dosEventLog.getSeverity())){
+        } else if ("Warning".equals(dosEventLog.getSeverity())) {
-            condition=100;
+            condition = 100;
-        }else if ("Major".equals(dosEventLog.getSeverity())){
+        } else if ("Major".equals(dosEventLog.getSeverity())) {
-            condition=250;
+            condition = 250;
-        }else if ("Severe".equals(dosEventLog.getSeverity())){
+        } else if ("Severe".equals(dosEventLog.getSeverity())) {
-            condition=500;
+            condition = 500;
-        }else if ("Critical".equals(dosEventLog.getSeverity())){
+        } else if ("Critical".equals(dosEventLog.getSeverity())) {
-            condition =800;
+            condition = 800;
         }
         switch (type) {
             case STATIC_CONDITION_TYPE:
                 return "Rate > " +
                         base + " " +
-                        tag + "/s" + "(>"+condition+"%)";
+                        tag + "/s" + "(>" + condition + "%)";
             case BASELINE_CONDITION_TYPE:
                 return tag + " > " +
                         percent + " of baseline";
@@ -276,8 +267,8 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
                 String[] ipArr = sourceIpList.split(",");
                 HashSet<String> countrySet = new HashSet<>();
                 for (String ip : ipArr) {
-                    String country = IpUtils.ipLookup.countryLookup(ip);
+                    String country = IpLookupUtils.getCountryLookup(ip);
-                    if (StringUtil.isNotBlank(country)){
+                    if (StringUtil.isNotBlank(country)) {
                         countrySet.add(country);
                     }
                 }
@@ -304,19 +295,13 @@ public class DosDetection extends BroadcastProcessFunction<DosSketchLog,Map<Stri
         return index;
     }
 
-    public static void main(String[] args) {
-//        System.out.println(new DosDetection().getSourceCountryList("192.0.2.3,138.199.14.31,255.255.255.255,121.14.89.209," +
-//                "23.200.74.224,161.117.68.253"));
-//        DosDetection dosDetection = new DosDetection();
-//        System.out.println(dosDetection.judgeSeverity(dosDetection.getDiffPercent(499, 1000)));
 
-    }
 
     private Double getDiffPercent(long diff, long base) {
         try {
             return BigDecimal.valueOf((float) diff / base).setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue();
-        }catch (Exception e){
+        } catch (Exception e) {
-            logger.info("当前阈值为0,进行下一阈值条件判断",e);
+            logger.info("当前阈值为0,进行下一阈值条件判断", e);
             return 0.0;
         }
 

src/main/java/com/zdjizhi/etl/EtlProcessFunction.java

@@ -2,7 +2,7 @@ package com.zdjizhi.etl;
 
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.common.DosSketchLog;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.api.java.tuple.Tuple3;
@@ -83,14 +83,14 @@ public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosS
                     endTime = newSketchLog.getSketch_start_time() > endTime ? newSketchLog.getSketch_start_time() : endTime;
                     duration = endTime - startTime == 0 ? 5 : endTime - startTime;
                 }else {
-                    if (sourceIpSet.size() < CommonConfig.SOURCE_IP_LIST_LIMIT){
+                    if (sourceIpSet.size() < FlowWriteConfig.SOURCE_IP_LIST_LIMIT){
                         sourceIpSet.add(sourceIp);
                     }
                 }
             }
             String sourceIpList = StringUtils.join(sourceIpSet, ",");
-            return Tuple6.of(sessions/CommonConfig.FLINK_WINDOW_MAX_TIME,packets/CommonConfig.FLINK_WINDOW_MAX_TIME,
+            return Tuple6.of(sessions/ FlowWriteConfig.FLINK_WINDOW_MAX_TIME,packets/ FlowWriteConfig.FLINK_WINDOW_MAX_TIME,
-                    bytes*8/CommonConfig.FLINK_WINDOW_MAX_TIME,sourceIpList,startTime,duration);
+                    bytes*8/ FlowWriteConfig.FLINK_WINDOW_MAX_TIME,sourceIpList,startTime,duration);
         }catch (Exception e){
             logger.error("聚合中间结果集失败 {}",e);
         }

src/main/java/com/zdjizhi/etl/ParseBaselineThreshold.java

@@ -2,9 +2,9 @@ package com.zdjizhi.etl;
 
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.CommonConfig;
+import com.geedgenetworks.utils.DateUtils;
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.common.DosBaselineThreshold;
-import com.zdjizhi.utils.DateUtils;
 import com.zdjizhi.utils.HbaseUtils;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
@@ -17,7 +17,7 @@ import java.util.*;
 
 public class ParseBaselineThreshold {
 
-//    private static final Logger logger = LoggerFactory.getLogger(ParseBaselineThreshold.class);
+
     private static final Log logger = LogFactory.get();
     private static ArrayList<String> floodTypeList = new ArrayList<>();
 
@@ -34,23 +34,23 @@ public class ParseBaselineThreshold {
     private static void prepareHbaseEnv() throws IOException {
         org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create();
 
-        config.set("hbase.zookeeper.quorum", CommonConfig.HBASE_ZOOKEEPER_QUORUM);
+        config.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_QUORUM);
         config.set("hbase.client.retries.number", "3");
         config.set("hbase.bulkload.retries.number", "3");
         config.set("zookeeper.recovery.retry", "3");
         config.set("hbase.defaults.for.version", "2.2.3");
         config.set("hbase.defaults.for.version.skip", "true");
-        config.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, CommonConfig.HBASE_CLIENT_OPERATION_TIMEOUT);
+        config.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, FlowWriteConfig.HBASE_CLIENT_OPERATION_TIMEOUT);
-        config.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, CommonConfig.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
+        config.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, FlowWriteConfig.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
 
-        TableName tableName = TableName.valueOf(CommonConfig.HBASE_BASELINE_TABLE_NAME);
+        TableName tableName = TableName.valueOf(FlowWriteConfig.HBASE_BASELINE_TABLE_NAME);
         Connection conn = ConnectionFactory.createConnection(config);
         table = conn.getTable(tableName);
         long currentTimeMillis = System.currentTimeMillis();
         scan = new Scan()
                 .setAllowPartialResults(true)
-                .setTimeRange(DateUtils.getSomeDate(new Date(currentTimeMillis), Math.negateExact(CommonConfig.HBASE_BASELINE_TTL)).getTime(), currentTimeMillis)
+                .setTimeRange(DateUtils.getSomeDate(new Date(currentTimeMillis), Math.negateExact(FlowWriteConfig.HBASE_BASELINE_TTL)).getTime(), currentTimeMillis)
-                .setLimit(CommonConfig.HBASE_BASELINE_TOTAL_NUM);
+                .setLimit(FlowWriteConfig.HBASE_BASELINE_TOTAL_NUM);
         logger.info("连接hbase成功，正在读取baseline数据");
     }
 
@@ -84,29 +84,4 @@ public class ParseBaselineThreshold {
         }
         return baselineMap;
     }
-
-    public static void main(String[] args) {
-        long currentTimeMillis = System.currentTimeMillis();
-        long p200D = DateUtils.getSomeDate(new Date(currentTimeMillis), Math.negateExact(CommonConfig.HBASE_BASELINE_TTL)).getTime();
-        System.out.println(p200D);
-        System.out.println(currentTimeMillis);
-        System.out.println(currentTimeMillis - p200D);
-
-
-        Map<String, Map<String, DosBaselineThreshold>> baselineMap = readFromHbase();
-        Set<String> keySet = baselineMap.keySet();
-        for (String key : keySet) {
-            Map<String, DosBaselineThreshold> stringTuple2Map = baselineMap.get(key);
-            Set<String> strings = stringTuple2Map.keySet();
-            for (String s:strings){
-                DosBaselineThreshold dosBaselineThreshold = stringTuple2Map.get(s);
-                System.out.println(key+"---"+s+"---"+dosBaselineThreshold);
-            }
-        }
-        System.out.println(baselineMap.size());
-    }
-
-
-
-
 }

src/main/java/com/zdjizhi/etl/ParseSketchLog.java

@@ -1,13 +1,11 @@
 package com.zdjizhi.etl;
 
 import com.alibaba.fastjson2.JSONObject;
-import com.fasterxml.jackson.databind.JavaType;
+import com.geedgenetworks.utils.StringUtil;
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.common.DosSketchLog;
 import com.zdjizhi.source.DosSketchSource;
 import com.zdjizhi.utils.FlinkEnvironmentUtils;
-//import com.zdjizhi.utils.JsonMapper;
-import com.zdjizhi.utils.StringUtil;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
@@ -24,22 +22,19 @@ import java.util.*;
 public class ParseSketchLog {
 
     private static Logger logger = LoggerFactory.getLogger(ParseSketchLog.class);
-//    private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();
-//    private static JavaType hashmapJsonType = jsonMapperInstance.createCollectionType(HashMap.class, String.class, Object.class);
-//    private static JavaType listType = jsonMapperInstance.createCollectionType(ArrayList.class, HashMap.class);
 
 
-    public static SingleOutputStreamOperator<DosSketchLog> getSketchSource(){
+    public static SingleOutputStreamOperator<DosSketchLog> getSketchSource() {
         return flatSketchSource().assignTimestampsAndWatermarks(createWatermarkStrategy());
     }
 
-    private static SingleOutputStreamOperator<DosSketchLog> flatSketchSource(){
+    private static SingleOutputStreamOperator<DosSketchLog> flatSketchSource() {
         return DosSketchSource.createDosSketchSource().flatMap(new FlatSketchLog());
     }
 
-    private static WatermarkStrategy<DosSketchLog> createWatermarkStrategy(){
+    private static WatermarkStrategy<DosSketchLog> createWatermarkStrategy() {
         return WatermarkStrategy
-                .<DosSketchLog>forBoundedOutOfOrderness(Duration.ofSeconds(CommonConfig.FLINK_WATERMARK_MAX_ORDERNESS))
+                .<DosSketchLog>forBoundedOutOfOrderness(Duration.ofSeconds(FlowWriteConfig.FLINK_WATERMARK_MAX_ORDERNESS))
                 .withTimestampAssigner((event, timestamp) -> event.getSketch_start_time() * 1000);
     }
 
@@ -47,17 +42,15 @@ public class ParseSketchLog {
         @Override
         public void flatMap(String s, Collector<DosSketchLog> collector) {
             try {
-                if (StringUtil.isNotBlank(s)){
+                if (StringUtil.isNotBlank(s)) {
                     HashMap<String, Object> sketchSource = JSONObject.parseObject(s, HashMap.class);
-//                    HashMap<String, Object> sketchSource = jsonMapperInstance.fromJson(s, hashmapJsonType);
                     long sketchStartTime = Long.parseLong(sketchSource.get("sketch_start_time").toString());
                     long sketchDuration = Long.parseLong(sketchSource.get("sketch_duration").toString());
                     String attackType = sketchSource.get("attack_type").toString();
                     int vsysId = Integer.parseInt(sketchSource.getOrDefault("common_vsys_id", 1).toString());
                     String report_ip_list = JSONObject.toJSONString(sketchSource.get("report_ip_list"));
                     ArrayList<HashMap<String, Object>> reportIpList = JSONObject.parseObject(report_ip_list, ArrayList.class);
-//                    ArrayList<HashMap<String, Object>> reportIpList = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(sketchSource.get("report_ip_list")), listType);
+                   for (HashMap<String, Object> obj : reportIpList) {
-                    for (HashMap<String, Object> obj : reportIpList) {
                         DosSketchLog dosSketchLog = new DosSketchLog();
                         dosSketchLog.setSketch_start_time(sketchStartTime);
                         dosSketchLog.setSketch_duration(sketchDuration);
@@ -74,19 +67,12 @@ public class ParseSketchLog {
                         dosSketchLog.setSketch_packets(sketchPackets);
                         dosSketchLog.setSketch_bytes(sketchBytes);
                         collector.collect(dosSketchLog);
-                        logger.debug("数据解析成功：{}",dosSketchLog.toString());
+                        logger.debug("数据解析成功：{}", dosSketchLog.toString());
                     }
                 }
             } catch (Exception e) {
-                logger.error("数据解析错误：{} \n{}",s,e);
+                logger.error("数据解析错误：{} \n{}", s, e);
             }
         }
     }
-
-
-    public static void main(String[] args) throws Exception {
-        flatSketchSource().print();
-        FlinkEnvironmentUtils.streamExeEnv.execute();
-    }
-
 }

src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java

@@ -5,16 +5,18 @@ import cn.hutool.log.LogFactory;
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.fastjson2.JSONObject;
 //import com.fasterxml.jackson.databind.JavaType;
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.common.DosDetectionThreshold;
 import com.zdjizhi.common.DosVsysId;
 import com.zdjizhi.utils.HttpClientUtils;
 //import com.zdjizhi.utils.JsonMapper;
-import com.zdjizhi.utils.NacosUtils;
+
+import com.zdjizhi.utils.connections.nacos.NacosUtils;
 import inet.ipaddr.IPAddress;
 import inet.ipaddr.IPAddressString;
 import org.apache.flink.shaded.guava18.com.google.common.collect.Range;
 import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap;
+
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.message.BasicHeader;
 
@@ -28,15 +30,9 @@ import java.util.Map;
  * @author wlh
  */
 public class ParseStaticThreshold {
-//    private static Logger logger = LoggerFactory.getLogger(ParseStaticThreshold.class);
     private static final Log logger = LogFactory.get();
     private static String encryptpwd;
 
-//    private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();
-//    private static JavaType hashmapJsonType = jsonMapperInstance.createCollectionType(HashMap.class, String.class, Object.class);
-//    private static JavaType thresholdType = jsonMapperInstance.createCollectionType(ArrayList.class, DosDetectionThreshold.class);
-//    private static JavaType vsysIDType = jsonMapperInstance.createCollectionType(ArrayList.class, DosVsysId.class);
-
     static {
         //加载加密登录密码
         encryptpwd = getEncryptpwd();
@@ -48,20 +44,18 @@ public class ParseStaticThreshold {
     private static String getEncryptpwd() {
         String psw = HttpClientUtils.ERROR_MESSAGE;
         try {
-            URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
+            URIBuilder uriBuilder = new URIBuilder(FlowWriteConfig.BIFANG_SERVER_URI);
             HashMap<String, Object> parms = new HashMap<>();
             parms.put("password", "admin");
-            HttpClientUtils.setUrlWithParams(uriBuilder, CommonConfig.BIFANG_SERVER_ENCRYPTPWD_PATH, parms);
+            HttpClientUtils.setUrlWithParams(uriBuilder, FlowWriteConfig.BIFANG_SERVER_ENCRYPTPWD_PATH, parms);
             String resposeJsonStr = HttpClientUtils.httpGet(uriBuilder.build());
             if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)) {
                 HashMap<String, Object> resposeMap = JSONObject.parseObject(resposeJsonStr, HashMap.class);
-//                HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
                 boolean success = (boolean) resposeMap.get("success");
                 String msg = resposeMap.get("msg").toString();
 
                 if (success) {
                     HashMap<String, Object> data = JSONObject.parseObject(JSONObject.toJSONString(resposeMap.get("data")), HashMap.class);
-//                    HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
                     psw = data.get("encryptpwd").toString();
                 } else {
                     logger.error(msg);
@@ -75,40 +69,6 @@ public class ParseStaticThreshold {
         return psw;
     }
 
-    /**
-     * 登录bifang服务，获取token
-     *
-     * @return token
-     */
-    private static String loginBifangServer() {
-        String token = HttpClientUtils.ERROR_MESSAGE;
-        try {
-            if (!HttpClientUtils.ERROR_MESSAGE.equals(encryptpwd)) {
-                URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
-                HashMap<String, Object> parms = new HashMap<>();
-                parms.put("username", "admin");
-                parms.put("password", encryptpwd);
-                HttpClientUtils.setUrlWithParams(uriBuilder, CommonConfig.BIFANG_SERVER_LOGIN_PATH, parms);
-                String resposeJsonStr = HttpClientUtils.httpPost(uriBuilder.build(), null);
-                if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)) {
-                    HashMap<String, Object> resposeMap = JSONObject.parseObject(resposeJsonStr, HashMap.class);
-//                    HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
-                    boolean success = (boolean) resposeMap.get("success");
-                    String msg = resposeMap.get("msg").toString();
-                    if (success) {
-                        HashMap<String, Object> data = JSONObject.parseObject(JSONObject.toJSONString(resposeMap.get("data")), HashMap.class);
-//                        HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
-                        token = data.get("token").toString();
-                    } else {
-                        logger.error(msg);
-                    }
-                }
-            }
-        } catch (Exception e) {
-            logger.error("登录失败,未获取到token ", e);
-        }
-        return token;
-    }
 
     /**
      * 获取vsysId配置列表
@@ -118,12 +78,12 @@ public class ParseStaticThreshold {
     private static ArrayList<DosVsysId> getVsysId() {
         ArrayList<DosVsysId> vsysIdList = null;
         try {
-            URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
+            URIBuilder uriBuilder = new URIBuilder(FlowWriteConfig.BIFANG_SERVER_URI);
             HashMap<String, Object> parms = new HashMap<>();
-            parms.put("pageSize", -1);
+            parms.put("page_size", -1);
 //            parms.put("orderBy", "vsysId desc");
             parms.put("type", 1);
-            HttpClientUtils.setUrlWithParams(uriBuilder, CommonConfig.BIFANG_SERVER_POLICY_VSYSID_PATH, parms);
+            HttpClientUtils.setUrlWithParams(uriBuilder, FlowWriteConfig.BIFANG_SERVER_POLICY_VSYSID_PATH, parms);
             String token = NacosUtils.getStringProperty("bifang.server.token");
             if (!HttpClientUtils.ERROR_MESSAGE.equals(token)) {
                 BasicHeader authorization = new BasicHeader("Authorization", token);
@@ -131,21 +91,15 @@ public class ParseStaticThreshold {
                 String resposeJsonStr = HttpClientUtils.httpGet(uriBuilder.build(), authorization, authorization1);
                 if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)) {
                     HashMap<String, Object> resposeMap = JSONObject.parseObject(resposeJsonStr, HashMap.class);
-//                    HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
                     boolean success = (boolean) resposeMap.get("success");
                     String msg = resposeMap.get("msg").toString();
                     if (success) {
                         HashMap<String, Object> data = JSONObject.parseObject(JSONObject.toJSONString(resposeMap.get("data")), HashMap.class);
-//                        HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
+                       Object list = data.get("list");
-                        Object list = data.get("list");
                         if (list != null) {
-                            String s = JSONObject.toJSONString(list);
                             List<DosVsysId> dosVsysIds = JSON.parseArray(JSONObject.toJSONString(list), DosVsysId.class);
-//                            vsysIdList= JSONObject.parseObject(JSONObject.toJSONString(list), DosVsysId.class);
                             vsysIdList= (ArrayList)dosVsysIds;
-
+                          logger.info("获取到vsysId {}条", vsysIdList.size());
-//                            vsysIdList = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(list), vsysIDType);
-                            logger.info("获取到vsysId {}条", vsysIdList.size());
                         } else {
                             logger.warn("vsysIdList为空");
                         }
@@ -172,35 +126,37 @@ public class ParseStaticThreshold {
             if (vsysIds != null) {
                 for (DosVsysId dosVsysId : vsysIds) {
                     Integer vsysId = dosVsysId.getId() == null ? 1 : dosVsysId.getId();
-                    Integer[] superiorIds = dosVsysId.getSuperiorIds();
+                    Integer[] superiorIds = dosVsysId.getSuperior_ids();
-                    URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
+                    URIBuilder uriBuilder = new URIBuilder(FlowWriteConfig.BIFANG_SERVER_URI);
                     HashMap<String, Object> parms = new HashMap<>();
-                    parms.put("pageSize", -1);
+                    parms.put("page_size", -1);
-                    parms.put("orderBy", "profileId asc");
+                    parms.put("order_by", "profileId asc");
-                    parms.put("isValid", 1);
+                    parms.put("is_valid", 1);
-                    parms.put("vsysId", vsysId);
+                    parms.put("vsys_id", vsysId);
-                    HttpClientUtils.setUrlWithParams(uriBuilder, CommonConfig.BIFANG_SERVER_POLICY_THRESHOLD_PATH, parms);
+                    HttpClientUtils.setUrlWithParams(uriBuilder, FlowWriteConfig.BIFANG_SERVER_POLICY_THRESHOLD_PATH, parms);
                     String token = NacosUtils.getStringProperty("bifang.server.token");
                     if (!HttpClientUtils.ERROR_MESSAGE.equals(token)) {
                         BasicHeader authorization = new BasicHeader("Authorization", token);
                         BasicHeader authorization1 = new BasicHeader("Content-Type", "application/x-www-form-urlencoded");
                         String resposeJsonStr = HttpClientUtils.httpGet(uriBuilder.build(), authorization, authorization1);
                         if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)) {
-//                            HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
+
                             HashMap<String, Object> resposeMap = JSONObject.parseObject(resposeJsonStr,HashMap.class);
+
+
                             boolean success = (boolean) resposeMap.get("success");
                             String msg = resposeMap.get("msg").toString();
                             if (success) {
-//                                HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
+                               HashMap<String, Object> data = JSONObject.parseObject(JSONObject.toJSONString(resposeMap.get("data")), HashMap.class);
-                                HashMap<String, Object> data = JSONObject.parseObject(JSONObject.toJSONString(resposeMap.get("data")), HashMap.class);
                                 Object list = data.get("list");
+
                                 if (list != null) {
-//                                    ArrayList<DosDetectionThreshold> thresholds = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(list), thresholdType);
+                                    System.out.println(list);
-//                                    ArrayList<DosDetectionThreshold> thresholds = JSONObject.parseObject(JSONObject.toJSONString(list), ArrayList.class);
+                                   List<DosDetectionThreshold> dosDetectionThresholds = JSON.parseArray(JSONObject.toJSONString(list), DosDetectionThreshold.class);
-                                    List<DosDetectionThreshold> dosDetectionThresholds = JSON.parseArray(JSONObject.toJSONString(list), DosDetectionThreshold.class);
+                                    System.out.println(dosDetectionThresholds);
                                     ArrayList<DosDetectionThreshold> thresholds = (ArrayList)dosDetectionThresholds;
                                     for (DosDetectionThreshold dosDetectionThreshold : thresholds) {
-                                        dosDetectionThreshold.setSuperiorIds(superiorIds);
+                                        dosDetectionThreshold.setSuperior_ids(superiorIds);
                                         vsysThresholds.add(dosDetectionThreshold);
                                     }
                                     logger.info("获取到vsys id是{}静态阈值配置{}条", vsysId, thresholds.size());
@@ -217,7 +173,6 @@ public class ParseStaticThreshold {
         } catch (Exception e) {
             logger.error("获取静态阈值配置失败,请检查bifang服务或登录配置信息 ", e);
         }
-
         return vsysThresholds;
     }
 
@@ -230,14 +185,19 @@ public class ParseStaticThreshold {
         HashMap<Integer, HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> thresholdRangeMap = new HashMap<>(4);
         try {
             ArrayList<DosDetectionThreshold> dosDetectionThreshold = getDosDetectionThreshold();
+
             if (dosDetectionThreshold != null && !dosDetectionThreshold.isEmpty()) {
                 for (DosDetectionThreshold threshold : dosDetectionThreshold) {
-                    String attackType = threshold.getAttackType();
+
-                    int vsysId = threshold.getVsysId();
+                    String attackType = threshold.getAttack_type();
+                    int vsysId = threshold.getVsys_id();
                     HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>> rangeMap = thresholdRangeMap.getOrDefault(vsysId, new HashMap<>());
 
                     TreeRangeMap<IPAddress, DosDetectionThreshold> treeRangeMap = rangeMap.getOrDefault(attackType, TreeRangeMap.create());
-                    ArrayList<String> serverIpList = threshold.getServerIpList();
+
+
+                    ArrayList<String> serverIpList = threshold.getServer_ip_list();
+
                     for (String sip : serverIpList) {
                         IPAddressString ipAddressString = new IPAddressString(sip);
                         if (ipAddressString.isIPAddress()) {
@@ -279,33 +239,38 @@ public class ParseStaticThreshold {
         return thresholdRangeMap;
     }
 
-    public static void main(String[] args) {
-        /*
-        ArrayList<DosDetectionThreshold> dosDetectionThreshold = getDosDetectionThreshold();
-//        dosDetectionThreshold.forEach(System.out::println);
-        getVsysId().forEach(System.out::println);
-        System.out.println("------------------------");
-        */
-        HashMap<Integer, HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> staticThreshold = createStaticThreshold();
 
-        System.out.println("------------------------");
 
-        for (Integer integer : staticThreshold.keySet()) {
+    /**
-            HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>> stringTreeRangeMapHashMap = staticThreshold.get(integer);
+     * 登录bifang服务，获取token
-            for (String type : stringTreeRangeMapHashMap.keySet()) {
+     *
-                Map<Range<IPAddress>, DosDetectionThreshold> asMapOfRanges = stringTreeRangeMapHashMap.get(type).asMapOfRanges();
+     * @return token
-                for (Range<IPAddress> range : asMapOfRanges.keySet()) {
+     */
-                    DosDetectionThreshold threshold = asMapOfRanges.get(range);
+    private static String loginBifangServer() {
-                    System.out.println(integer + "---" + type + "---" + range + "---" + threshold);
+        String token = HttpClientUtils.ERROR_MESSAGE;
+        try {
+            if (!HttpClientUtils.ERROR_MESSAGE.equals(encryptpwd)) {
+                URIBuilder uriBuilder = new URIBuilder(FlowWriteConfig.BIFANG_SERVER_URI);
+                HashMap<String, Object> parms = new HashMap<>();
+                parms.put("username", "admin");
+                parms.put("password", encryptpwd);
+                HttpClientUtils.setUrlWithParams(uriBuilder, FlowWriteConfig.BIFANG_SERVER_LOGIN_PATH, parms);
+                String resposeJsonStr = HttpClientUtils.httpPost(uriBuilder.build(), null);
+                if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)) {
+                    HashMap<String, Object> resposeMap = JSONObject.parseObject(resposeJsonStr, HashMap.class);
+                    boolean success = (boolean) resposeMap.get("success");
+                    String msg = resposeMap.get("msg").toString();
+                    if (success) {
+                        HashMap<String, Object> data = JSONObject.parseObject(JSONObject.toJSONString(resposeMap.get("data")), HashMap.class);
+                        token = data.get("token").toString();
+                    } else {
+                        logger.error(msg);
+                    }
                 }
-                System.out.println("------------------------");
             }
-
+        } catch (Exception e) {
+            logger.error("登录失败,未获取到token ", e);
         }
-//        String s = loginBifangServer();
+        return token;
-//        System.out.println(s);
-
     }
-
-
 }

src/main/java/com/zdjizhi/etl/TrafficServerIpMetrics.java

@@ -2,7 +2,7 @@ package com.zdjizhi.etl;
 
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.common.DosMetricsLog;
 import com.zdjizhi.common.DosSketchLog;
 
@@ -26,16 +26,11 @@ class TrafficServerIpMetrics {
     }
 
     private static long timeFloor(long sketchStartTime){
-        return sketchStartTime / CommonConfig.FLINK_WINDOW_MAX_TIME * CommonConfig.FLINK_WINDOW_MAX_TIME;
+        return sketchStartTime / FlowWriteConfig.FLINK_WINDOW_MAX_TIME * FlowWriteConfig.FLINK_WINDOW_MAX_TIME;
     }
 
     private static int getPartitionNumByIp(String destinationIp){
-        return Math.abs(destinationIp.hashCode()) % CommonConfig.DESTINATION_IP_PARTITION_NUM;
+        return Math.abs(destinationIp.hashCode()) % FlowWriteConfig.DESTINATION_IP_PARTITION_NUM;
-    }
-
-    public static void main(String[] args) {
-        System.out.println(getPartitionNumByIp("146.177.223.43"));
-        System.out.println("146.177.223.43".hashCode());
     }
 
 }

src/main/java/com/zdjizhi/sink/DosEventSink.java

@@ -1,7 +1,7 @@
 package com.zdjizhi.sink;
 
 import com.alibaba.fastjson2.JSONObject;
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.common.DosEventLog;
 //import com.zdjizhi.utils.JsonMapper;
 import com.zdjizhi.utils.KafkaUtils;
@@ -16,8 +16,8 @@ class DosEventSink {
                 .filter(Objects::nonNull)
 //                .map(JsonMapper::toJsonString)
                 .map(JSONObject::toJSONString)
-                .addSink(KafkaUtils.getKafkaSink(CommonConfig.KAFKA_OUTPUT_EVENT_TOPIC_NAME))
+                .addSink(KafkaUtils.getKafkaSink(FlowWriteConfig.KAFKA_OUTPUT_EVENT_TOPIC_NAME))
-                .setParallelism(CommonConfig.KAFKA_OUTPUT_EVENT_PARALLELISM);
+                .setParallelism(FlowWriteConfig.KAFKA_OUTPUT_EVENT_PARALLELISM);
     }
 
 }

src/main/java/com/zdjizhi/sink/OutputStreamSink.java

@@ -3,18 +3,14 @@ package com.zdjizhi.sink;
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
 import com.alibaba.nacos.api.PropertyKeyConst;
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.common.DosEventLog;
 import com.zdjizhi.common.DosMetricsLog;
 import com.zdjizhi.common.DosSketchLog;
 import com.zdjizhi.etl.DosDetection;
 import com.zdjizhi.etl.EtlProcessFunction;
 import com.zdjizhi.etl.ParseSketchLog;
-import com.zdjizhi.source.DosSketchSource;
 import com.zdjizhi.utils.FlinkEnvironmentUtils;
-import org.apache.flink.api.common.state.MapStateDescriptor;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeinfo.Types;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.streaming.api.datastream.*;
@@ -22,7 +18,6 @@ import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindo
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.util.OutputTag;
 
-import java.util.Map;
 import java.util.Properties;
 
 /**
@@ -39,44 +34,24 @@ public class OutputStreamSink {
             SingleOutputStreamOperator<DosSketchLog> middleStream = getMiddleStream();
             DosEventSink.dosEventOutputSink(getEventSinkStream(middleStream));
             TrafficServerIpMetricsSink.sideOutputMetricsSink(middleStream);
-            FlinkEnvironmentUtils.streamExeEnv.execute(CommonConfig.STREAM_EXECUTION_JOB_NAME);
+            FlinkEnvironmentUtils.streamExeEnv.execute(FlowWriteConfig.STREAM_EXECUTION_JOB_NAME);
         } catch (Exception e) {
             logger.error("任务启动失败 {}",e);
         }
     }
 
     private static SingleOutputStreamOperator<DosEventLog> getEventSinkStream(SingleOutputStreamOperator<DosSketchLog> middleStream){
-        DataStreamSource<Map<String, String>> broadcastSource=null;
-        Properties nacosProperties = new Properties();
-
-        nacosProperties.put(PropertyKeyConst.SERVER_ADDR,CommonConfig.NACOS_SERVER_ADDR);
-        nacosProperties.setProperty(PropertyKeyConst.USERNAME, CommonConfig.NACOS_USERNAME);
-        nacosProperties.setProperty(PropertyKeyConst.PASSWORD, CommonConfig.NACOS_PASSWORD);
-        nacosProperties.setProperty(PropertyKeyConst.NAMESPACE, CommonConfig.NACOS_NAMESPACE);
-
-        if ("CLUSTER".equals(CommonConfig.CLUSTER_OR_SINGLE)){
-            broadcastSource = DosSketchSource.broadcastSource(nacosProperties);
-        }else {
-            broadcastSource= DosSketchSource.singleBroadcastSource(nacosProperties);
-        }
-
-        MapStateDescriptor<String,Map> descriptor =
-                new MapStateDescriptor<>("descriptorTest", Types.STRING, TypeInformation.of(Map.class));
-
-        BroadcastStream<Map<String, String>> broadcast = broadcastSource.broadcast(descriptor);
-
         return middleStream
-                .connect(broadcast)
+                .process(new DosDetection()).setParallelism(FlowWriteConfig.FLINK_DETECTION_MAP_PARALLELISM);
-                .process(new DosDetection()).setParallelism(CommonConfig.FLINK_DETECTION_MAP_PARALLELISM);
 
     }
 
     private static SingleOutputStreamOperator<DosSketchLog> getMiddleStream(){
         return ParseSketchLog.getSketchSource()
                 .keyBy(new KeysSelector())
-                .window(TumblingEventTimeWindows.of(Time.seconds(CommonConfig.FLINK_WINDOW_MAX_TIME)))
+                .window(TumblingEventTimeWindows.of(Time.seconds(FlowWriteConfig.FLINK_WINDOW_MAX_TIME)))
                 .process(new EtlProcessFunction())
-                .setParallelism(CommonConfig.FLINK_FIRST_AGG_PARALLELISM);
+                .setParallelism(FlowWriteConfig.FLINK_FIRST_AGG_PARALLELISM);
     }
 
     private static class KeysSelector implements KeySelector<DosSketchLog, Tuple3<String, String, Integer>>{

src/main/java/com/zdjizhi/sink/TrafficServerIpMetricsSink.java

@@ -1,10 +1,10 @@
 package com.zdjizhi.sink;
 
 import com.alibaba.fastjson2.JSONObject;
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.common.DosMetricsLog;
 import com.zdjizhi.common.DosSketchLog;
-import com.zdjizhi.utils.JsonMapper;
+
 import com.zdjizhi.utils.KafkaUtils;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
@@ -16,10 +16,8 @@ class TrafficServerIpMetricsSink {
     static void sideOutputMetricsSink(SingleOutputStreamOperator<DosSketchLog> outputStream){
         DataStream<DosMetricsLog> sideOutput = outputStream.getSideOutput(outputTag);
 //        sideOutput.map(JsonMapper::toJsonString).addSink(KafkaUtils.getKafkaSink(CommonConfig.KAFKA_OUTPUT_METRIC_TOPIC_NAME))
-        sideOutput.map(JSONObject::toJSONString).addSink(KafkaUtils.getKafkaSink(CommonConfig.KAFKA_OUTPUT_METRIC_TOPIC_NAME))
+        sideOutput.map(JSONObject::toJSONString).addSink(KafkaUtils.getKafkaSink(FlowWriteConfig.KAFKA_OUTPUT_METRIC_TOPIC_NAME))
-                .setParallelism(CommonConfig.KAFKA_OUTPUT_METRIC_PARALLELISM);
+                .setParallelism(FlowWriteConfig.KAFKA_OUTPUT_METRIC_PARALLELISM);
-
-
 
 
     }

src/main/java/com/zdjizhi/source/DosSketchSource.java

@@ -1,15 +1,14 @@
 package com.zdjizhi.source;
 
-import com.zdjizhi.common.CommonConfig;
+
-import com.zdjizhi.common.CustomFile;
+
+import com.zdjizhi.common.FlowWriteConfig;
 import com.zdjizhi.utils.FlinkEnvironmentUtils;
 import org.apache.flink.api.common.serialization.SimpleStringSchema;
 import org.apache.flink.streaming.api.datastream.DataStreamSource;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
 
-import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 
 /**
@@ -21,26 +20,18 @@ public class DosSketchSource {
 
     public static DataStreamSource<String> createDosSketchSource(){
         Properties properties = new Properties();
-        properties.setProperty("bootstrap.servers", CommonConfig.KAFKA_INPUT_BOOTSTRAP_SERVERS);
+        properties.setProperty("bootstrap.servers", FlowWriteConfig.KAFKA_INPUT_BOOTSTRAP_SERVERS);
-        properties.setProperty("group.id", CommonConfig.KAFKA_GROUP_ID);
+        properties.setProperty("group.id", FlowWriteConfig.KAFKA_GROUP_ID);
-        if (CommonConfig.SASL_JAAS_CONFIG_FLAG == 1){
+        if (FlowWriteConfig.SASL_JAAS_CONFIG_FLAG == 1){
             properties.put("security.protocol", "SASL_PLAINTEXT");
             properties.put("sasl.mechanism", "PLAIN");
-            properties.put("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\""+CommonConfig.SASL_JAAS_CONFIG_USER+"\" password=\""+CommonConfig.SASL_JAAS_CONFIG_PASSWORD+"\";");
+            properties.put("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\""+ FlowWriteConfig.SASL_JAAS_CONFIG_USER+"\" password=\""+ FlowWriteConfig.SASL_JAAS_CONFIG_PASSWORD+"\";");
         }
 
         return streamExeEnv.addSource(new FlinkKafkaConsumer<String>(
-                CommonConfig.KAFKA_INPUT_TOPIC_NAME,
+                FlowWriteConfig.KAFKA_INPUT_TOPIC_NAME,
                 new SimpleStringSchema(), properties))
-                .setParallelism(CommonConfig.KAFKA_INPUT_PARALLELISM);
+                .setParallelism(FlowWriteConfig.KAFKA_INPUT_PARALLELISM);
     }
 
-
-    public static DataStreamSource<Map<String, String>> broadcastSource(Properties nacosProperties){
-        return streamExeEnv.addSource(new HttpSource(nacosProperties, CommonConfig.NACOS_DATA_ID, CommonConfig.NACOS_GROUP, CommonConfig.NACOS_READ_TIMEOUT));
-    }
-
-    public static DataStreamSource<Map<String, String>>singleBroadcastSource(Properties nacosProperties){
-        return streamExeEnv.addSource(new SingleHttpSource(nacosProperties, CommonConfig.NACOS_DATA_ID, CommonConfig.NACOS_GROUP, CommonConfig.NACOS_READ_TIMEOUT));
-    }
 }

src/main/java/com/zdjizhi/source/HttpSource.java

@@ -1,260 +0,0 @@
-package com.zdjizhi.source;
-
-import cn.hutool.core.io.FileUtil;
-import cn.hutool.core.io.IoUtil;
-import cn.hutool.core.io.file.FileReader;
-import cn.hutool.crypto.digest.DigestUtil;
-import cn.hutool.json.JSONObject;
-import com.alibaba.nacos.api.NacosFactory;
-import com.alibaba.nacos.api.PropertyKeyConst;
-import com.alibaba.nacos.api.config.ConfigService;
-import com.alibaba.nacos.api.config.listener.Listener;
-import com.alibaba.nacos.api.exception.NacosException;
-import com.fasterxml.jackson.databind.JavaType;
-import com.google.common.base.Joiner;
-import com.jayway.jsonpath.JsonPath;
-import com.zdjizhi.common.CommonConfig;
-import com.zdjizhi.common.CustomFile;
-import com.zdjizhi.common.KnowledgeLog;
-import com.zdjizhi.utils.*;
-import org.apache.commons.io.IOUtils;
-import org.apache.flink.configuration.Configuration;
-import org.apache.http.Header;
-import org.apache.http.message.BasicHeader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.Executor;
-
-
-public class HttpSource extends RichHttpSourceFunction<Map<String, String>> {
-    private static final Logger logger = LoggerFactory.getLogger(HttpSource.class);
-
-    private static final String EXPR = "$.[?(@.version=='latest' && @.name in ['ip_v4_built_in','ip_v6_built_in','ip_v4_user_defined','ip_v6_user_defined'])].['name','sha256','format','path']";
-
-    private static Map<String, String> knowledgeMetaCache = new HashMap<>();
-
-    private static HashMap<String, String> knowledgeUpdateCache;
-
-    private static final int TRY_TIMES = 3;
-
-    private static HttpClientUtils2 httpClientUtils;
-
-    //连接nacos的配置
-    private Properties nacosProperties;
-
-    //nacos data id
-    private String NACOS_DATA_ID;
-
-    //nacos group
-    private String NACOS_GROUP;
-
-    //nacos 连接超时时间
-    private long NACOS_READ_TIMEOUT;
-
-    private ConfigService configService;
-
-    private static Header header;
-
-    //运行状态cancel时置为false
-    private boolean isRunning = true;
-    //是否下发，默认不发送
-    private boolean isSending = false;
-
-
-    public HttpSource(Properties nacosProperties, String NACOS_DATA_ID, String NACOS_GROUP, long NACOS_READ_TIMEOUT) {
-        this.nacosProperties = nacosProperties;
-        this.NACOS_DATA_ID = NACOS_DATA_ID;
-        this.NACOS_GROUP = NACOS_GROUP;
-        this.NACOS_READ_TIMEOUT = NACOS_READ_TIMEOUT;
-    }
-
-    @Override
-    public void open(Configuration parameters) throws Exception {
-        super.open(parameters);
-        httpClientUtils = new HttpClientUtils2();
-        //初始化元数据缓存
-        knowledgeMetaCache = new HashMap<>(16);
-        //初始化定位库缓存
-        knowledgeUpdateCache = new HashMap<>(16);
-
-        header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
-        //连接nacos配置
-        try {
-            configService = NacosFactory.createConfigService(nacosProperties);
-        }catch (NacosException e){
-            logger.error("Get Schema config from Nacos error,The exception message is :{}", e.getMessage());
-        }
-
-        //初始化知识库
-        initKnowledge();
-        logger.info("连接nacos：" + nacosProperties.getProperty(PropertyKeyConst.SERVER_ADDR));
-
-
-    }
-    @Override
-    public void run(SourceContext ctx) throws Exception {
-        if (!knowledgeUpdateCache.isEmpty()){
-            ctx.collect(knowledgeUpdateCache);
-                knowledgeUpdateCache.clear();
-        }
-//        }
-
-
-        configService.addListener(NACOS_DATA_ID, NACOS_GROUP, new Listener() {
-            @Override
-            public Executor getExecutor() {
-                return null;
-            }
-
-            @Override
-            public void receiveConfigInfo(String configMsg) {
-                try {
-                    logger.info("receive update config:" + configMsg);
-                    if (StringUtil.isNotBlank(configMsg)) {
-                        ArrayList<Object> metaList = JsonPath.parse(configMsg).read(EXPR);
-                        if (metaList.size() > 0) {
-                            for (Object metadata : metaList) {
-                                JSONObject knowledgeJson = new JSONObject(metadata, false, true);
-                                String fileName = Joiner.on(CommonConfig.LOCATION_SEPARATOR).useForNull("").join(knowledgeJson.getStr("name"),
-                                        knowledgeJson.getStr("format"));
-                                String sha256 = knowledgeJson.getStr("sha256");
-                                String filePath = knowledgeJson.getStr("path");
-                                if (!sha256.equals(knowledgeMetaCache.get(fileName))) {
-                                    knowledgeMetaCache.put(fileName, sha256);
-                                    updateKnowledge(fileName, filePath,sha256);
-                                }
-                            }
-                            if (!knowledgeUpdateCache.isEmpty()){
-                                ctx.collect(knowledgeUpdateCache);
-                                knowledgeUpdateCache.clear();
-                            }
-                        }
-                    }
-
-                } catch (Exception e) {
-                    logger.error("监听nacos配置失败", e);
-                }
-            }
-        });
-
-        while (isRunning) {
-            try {
-                Thread.sleep(10000);
-            }catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-
-    }
-
-    private void initKnowledge(){
-        String configMsg = "";
-        try {
-            configMsg=configService.getConfig(NACOS_DATA_ID, NACOS_GROUP, NACOS_READ_TIMEOUT);
-        } catch (NacosException e) {
-            logger.error("从Nacos获取知识库元数据配置文件异常，异常信息为:{}", e.getMessage());
-        }
-
-        if (StringUtil.isNotBlank(configMsg)){
-            ArrayList<Object> metaList = JsonPath.parse(configMsg).read(EXPR);
-            if (metaList.size() > 0) {
-                for (Object metadata : metaList) {
-                    JSONObject knowledgeJson = new JSONObject(metadata, false, true);
-                    String fileName = Joiner.on(CommonConfig.LOCATION_SEPARATOR).useForNull("").join(knowledgeJson.getStr("name"),
-                            knowledgeJson.getStr("format"));
-                    String sha256 = knowledgeJson.getStr("sha256");
-                    String filePath = knowledgeJson.getStr("path");
-                    byte[] localFileByte = getLocalFile(fileName);
-                    String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                    if (sha256.equals(localFileSha256Hex)){
-                        logger.info("本地文件{}的sha256为:{} ,Nacos内记录为:{} ,sha256相等", fileName, localFileSha256Hex, sha256);
-                        knowledgeMetaCache.put(fileName, sha256);
-                    }else {
-                        logger.info("本地文件{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等，更新本地文件及缓存", fileName, localFileSha256Hex, sha256);
-                        updateKnowledge(fileName,filePath,sha256);
-                    }
-
-                }
-            }
-        }
-    }
-
-    private void updateKnowledge(String fileName, String filePath,String sha256) {
-        InputStream inputStream = null;
-        int retryNum = 0;
-        try {
-            while (retryNum < TRY_TIMES){
-                inputStream = httpClientUtils.httpGetInputStream(filePath, 90000, header);
-                if (inputStream !=null){
-                    byte[] downloadBytes = IOUtils.toByteArray(inputStream);
-                    String downloadFileSha256Hex = DigestUtil.sha256Hex(downloadBytes);
-                    if (sha256.equals(downloadFileSha256Hex)&& downloadBytes.length > 0 ){
-                        logger.info("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256相等", fileName, sha256);
-                        boolean updateStatus = updateLocalFile(fileName, downloadBytes);
-                        if (updateStatus){
-                            knowledgeMetaCache.put(fileName,sha256);
-                            knowledgeUpdateCache.put(fileName, sha256);
-                            retryNum = TRY_TIMES;
-                        }else {
-                            retryNum++;
-                            //避免频繁请求HOS
-                            Thread.sleep(10000);
-                        }
-                    }else {
-                        logger.error("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等 开始第{}次重试下载文件", fileName, downloadFileSha256Hex, sha256, retryNum);
-                        retryNum++;
-                        //避免频繁请求HOS
-                        Thread.sleep(10000);
-                    }
-                }
-            }
-        } catch (IOException ioException) {
-            ioException.printStackTrace();
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        } finally {
-            IOUtils.closeQuietly(inputStream);
-        }
-    }
-
-
-    private boolean updateLocalFile(String fileName,byte[] downloadBytes) {
-        FileOutputStream outputStream = null;
-        boolean updateStatus = false;
-        try {
-            HdfsUtils.uploadFileByBytes(CommonConfig.HDFS_PATH + fileName, downloadBytes);
-            updateStatus=true;
-        } catch (IOException ioe) {
-            logger.error("更新本地文件{}时发生IO异常,异常信息为:", fileName, ioe.getMessage());
-            ioe.printStackTrace();
-        } catch (RuntimeException e) {
-            logger.error("更新本地文件{}时发生异常,异常信息为:", fileName, e.getMessage());
-            e.printStackTrace();
-        } finally {
-            IOUtils.closeQuietly(outputStream);
-        }
-        return updateStatus;
-    }
-
-    private static byte[] getLocalFile(String name) {
-        byte[] fileBytes = null;
-        try {
-            fileBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + name) ;
-        } catch (RuntimeException | IOException e) {
-            logger.error("IpLookupUtils download MMDB files error, message is:" + e.getMessage());
-            e.printStackTrace();
-        }
-        return fileBytes;
-    }
-
-    @Override
-    public void cancel() {
-        this.isRunning = false;
-    }
-
-
-}

src/main/java/com/zdjizhi/source/HttpSourceFunction.java

@@ -1,6 +0,0 @@
-package com.zdjizhi.source;
-
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-
-public interface HttpSourceFunction<OUT> extends SourceFunction<OUT> {
-}

src/main/java/com/zdjizhi/source/RichHttpSourceFunction.java

@@ -1,10 +0,0 @@
-package com.zdjizhi.source;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-
-public abstract class RichHttpSourceFunction<OUT> extends AbstractRichFunction implements HttpSourceFunction<OUT> {
-    private static final long serialVersionUID = 1L;
-
-    public RichHttpSourceFunction() {
-    }
-}

src/main/java/com/zdjizhi/source/SingleHttpSource.java

@@ -1,270 +0,0 @@
-package com.zdjizhi.source;
-
-import cn.hutool.core.io.FileUtil;
-import cn.hutool.core.io.IoUtil;
-import cn.hutool.core.io.file.FileReader;
-import cn.hutool.crypto.digest.DigestUtil;
-import cn.hutool.json.JSONObject;
-import com.alibaba.nacos.api.NacosFactory;
-import com.alibaba.nacos.api.PropertyKeyConst;
-import com.alibaba.nacos.api.config.ConfigService;
-import com.alibaba.nacos.api.config.listener.Listener;
-import com.alibaba.nacos.api.exception.NacosException;
-import com.fasterxml.jackson.databind.JavaType;
-import com.google.common.base.Joiner;
-import com.jayway.jsonpath.JsonPath;
-import com.zdjizhi.common.CommonConfig;
-import com.zdjizhi.common.CustomFile;
-import com.zdjizhi.common.KnowledgeLog;
-import com.zdjizhi.utils.*;
-import org.apache.commons.io.IOUtils;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.http.Header;
-import org.apache.http.message.BasicHeader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.Executor;
-
-public class SingleHttpSource extends RichHttpSourceFunction<Map<String, String>> {
-
-    private static final Logger logger = LoggerFactory.getLogger(SingleHttpSource.class);
-
-    private static final String EXPR = "$.[?(@.version=='latest' && @.name in ['ip_v4_built_in','ip_v6_built_in','ip_v4_user_defined','ip_v6_user_defined'])].['name','sha256','format','path']";
-
-
-    private static Map<String, String> knowledgeMetaCache = new HashMap<>();
-
-    private static HashMap<String, String> knowledgeUpdateCache;
-
-    private static final int TRY_TIMES = 3;
-
-    private Properties nacosProperties;
-
-    private String NACOS_DATA_ID;
-
-    private String NACOS_GROUP;
-
-    private long NACOS_READ_TIMEOUT;
-
-    private static HttpClientUtils2 httpClientUtils ;
-
-    private ConfigService configService;
-
-    private static Header header;
-
-    private boolean isRunning = true;
-
-
-
-    public SingleHttpSource(Properties nacosProperties, String NACOS_DATA_ID, String NACOS_GROUP, long NACOS_READ_TIMEOUT) {
-        this.nacosProperties = nacosProperties;
-        this.NACOS_DATA_ID = NACOS_DATA_ID;
-        this.NACOS_GROUP = NACOS_GROUP;
-        this.NACOS_READ_TIMEOUT = NACOS_READ_TIMEOUT;
-    }
-
-
-
-    @Override
-    public void open(Configuration parameters) throws Exception {
-        super.open(parameters);
-        httpClientUtils = new HttpClientUtils2();
-        //初始化元数据缓存
-        knowledgeMetaCache = new HashMap<>(16);
-        //初始化定位库缓存
-        knowledgeUpdateCache = new HashMap<>(16);
-
-        header = new BasicHeader("token", CommonConfig.HOS_TOKEN);
-
-        //连接nacos配置
-        try {
-            configService = NacosFactory.createConfigService(nacosProperties);
-        }catch (NacosException e){
-            logger.error("Get Schema config from Nacos error,The exception message is :{}", e.getMessage());
-        }
-
-        //初始化知识库
-        initKnowledge();
-        logger.info("连接nacos：" + nacosProperties.getProperty(PropertyKeyConst.SERVER_ADDR));
-    }
-
-    @Override
-    public void run(SourceContext ctx) throws Exception {
-        if (!knowledgeUpdateCache.isEmpty()){
-            ctx.collect(knowledgeUpdateCache);
-                knowledgeUpdateCache.clear();
-        }
-
-        configService.addListener(NACOS_DATA_ID, NACOS_GROUP, new Listener() {
-            @Override
-            public Executor getExecutor() {
-                return null;
-            }
-
-            @Override
-            public void receiveConfigInfo(String configMsg) {
-                try {
-                    logger.info("receive update config:" + configMsg);
-                    if (StringUtil.isNotBlank(configMsg)) {
-                        ArrayList<Object> metaList = JsonPath.parse(configMsg).read(EXPR);
-                        if (metaList.size() > 0) {
-                            for (Object metadata : metaList) {
-                                JSONObject knowledgeJson = new JSONObject(metadata, false, true);
-                                String fileName = Joiner.on(CommonConfig.LOCATION_SEPARATOR).useForNull("").join(knowledgeJson.getStr("name"),
-                                        knowledgeJson.getStr("format"));
-                                String sha256 = knowledgeJson.getStr("sha256");
-                                String filePath = knowledgeJson.getStr("path");
-                                if (!sha256.equals(knowledgeMetaCache.get(fileName))) {
-                                    knowledgeMetaCache.put(fileName, sha256);
-                                    updateKnowledge(fileName, filePath,sha256);
-                                }
-                            }
-                            if (!knowledgeUpdateCache.isEmpty()){
-                                ctx.collect(knowledgeUpdateCache);
-                                knowledgeUpdateCache.clear();
-                            }
-                        }
-                    }
-
-                } catch (Exception e) {
-                    logger.error("监听nacos配置失败", e);
-                }
-                System.out.println(configMsg);
-            }
-        });
-
-        while (isRunning) {
-            try {
-                Thread.sleep(10000);
-            }catch (InterruptedException e){
-                e.printStackTrace();
-            }
-
-        }
-
-
-
-    }
-
-
-
-    private void initKnowledge(){
-        String configMsg = "";
-        try {
-            configMsg=configService.getConfig(NACOS_DATA_ID, NACOS_GROUP, NACOS_READ_TIMEOUT);
-        } catch (NacosException e) {
-            logger.error("从Nacos获取知识库元数据配置文件异常，异常信息为:{}", e.getMessage());
-        }
-
-        if (StringUtil.isNotBlank(configMsg)){
-            ArrayList<Object> metaList = JsonPath.parse(configMsg).read(EXPR);
-            if (metaList.size() > 0) {
-                for (Object metadata : metaList) {
-                    JSONObject knowledgeJson = new JSONObject(metadata, false, true);
-                    String fileName = Joiner.on(CommonConfig.LOCATION_SEPARATOR).useForNull("").join(knowledgeJson.getStr("name"),
-                            knowledgeJson.getStr("format"));
-                    String sha256 = knowledgeJson.getStr("sha256");
-                    String filePath = knowledgeJson.getStr("path");
-                    byte[] localFileByte = getLocalFile(fileName);
-                    String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                    if (sha256.equals(localFileSha256Hex)){
-                        logger.info("本地文件{}的sha256为:{} ,Nacos内记录为:{} ,sha256相等", fileName, localFileSha256Hex, sha256);
-                        knowledgeMetaCache.put(fileName, sha256);
-                    }else {
-                        logger.info("本地文件{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等，更新本地文件及缓存", fileName, localFileSha256Hex, sha256);
-                        updateKnowledge(fileName,filePath,sha256);
-                    }
-
-                }
-            }
-        }
-    }
-
-
-
-    private void updateKnowledge(String fileName, String filePath,String sha256) {
-        InputStream inputStream = null;
-        int retryNum = 0;
-        try {
-            while (retryNum < TRY_TIMES){
-                inputStream = httpClientUtils.httpGetInputStream(filePath, 3000, header);
-                if (inputStream !=null){
-                    byte[] downloadBytes = IOUtils.toByteArray(inputStream);
-                    String downloadFileSha256Hex = DigestUtil.sha256Hex(downloadBytes);
-                    if (sha256.equals(downloadFileSha256Hex)&& downloadBytes.length > 0 ){
-                        logger.info("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256相等", fileName, sha256);
-                        boolean updateStatus = updateLocalFile(fileName, downloadBytes);
-                        if (updateStatus){
-                            knowledgeMetaCache.put(fileName,sha256);
-                            knowledgeUpdateCache.put(fileName, sha256);
-                            retryNum = TRY_TIMES;
-                        }else {
-                            retryNum++;
-                            //避免频繁请求HOS
-                            Thread.sleep(10000);
-                        }
-//                            isSending = true;
-                    }else {
-                        logger.error("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等 开始第{}次重试下载文件", fileName, downloadFileSha256Hex, sha256, retryNum);
-                        retryNum++;
-                        //避免频繁请求HOS
-                        Thread.sleep(10000);
-                    }
-                }
-            }
-
-        } catch (IOException ioException) {
-            ioException.printStackTrace();
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        } finally {
-            IOUtils.closeQuietly(inputStream);
-        }
-    }
-
-
-
-    private boolean updateLocalFile(String fileName,byte[] downloadBytes) {
-        FileOutputStream outputStream = null;
-        boolean updateStatus = false;
-        try {
-            FileUtil.mkdir(CommonConfig.DOWNLOAD_PATH);
-            File file = new File(CommonConfig.DOWNLOAD_PATH.concat(File.separator).concat(fileName));
-            outputStream = new FileOutputStream(file);
-            IoUtil.copy(new ByteArrayInputStream(downloadBytes), outputStream);
-            updateStatus=true;
-        } catch (IOException ioe) {
-            logger.error("更新本地文件{}时发生IO异常,异常信息为:", fileName, ioe.getMessage());
-            ioe.printStackTrace();
-        } catch (RuntimeException e) {
-            logger.error("更新本地文件{}时发生异常,异常信息为:", fileName, e.getMessage());
-            e.printStackTrace();
-        } finally {
-            IOUtils.closeQuietly(outputStream);
-        }
-        return updateStatus;
-    }
-
-    private static byte[] getLocalFile(String name) {
-        byte[] fileBytes = null;
-        try {
-            fileBytes=new FileReader(CommonConfig.DOWNLOAD_PATH + name).readBytes();
-        } catch (RuntimeException e) {
-            logger.error("IpLookupUtils download MMDB files error, message is:" + e.getMessage());
-            e.printStackTrace();
-        }
-        return fileBytes;
-    }
-
-
-    @Override
-    public void cancel() {
-        this.isRunning = false;
-    }
-}
-

src/main/java/com/zdjizhi/utils/CollectionUtils.java

@@ -1,23 +0,0 @@
-package com.zdjizhi.utils;
-
-import java.util.Collection;
-import java.util.HashSet;
-
-/**
- * @author wlh
- * 扩展集合处理工具
- */
-public class CollectionUtils {
-
-    public static<T> Collection<T> takeUniqueLimit(Collection<T> collection, int limit){
-        int i =0;
-        Collection<T> newSet = new HashSet<>();
-        for (T t:collection){
-            if (i < limit){
-                newSet.add(t);
-                i += 1;
-            }
-        }
-        return newSet;
-    }
-}

src/main/java/com/zdjizhi/utils/FileByteUtils.java

@@ -1,24 +0,0 @@
-package com.zdjizhi.utils;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-
-public class FileByteUtils {
-
-    public static byte[] getFileBytes (String filePath) throws IOException {
-        File file = new File(filePath);
-        FileInputStream fis = new FileInputStream(file);
-        ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
-        byte[] b = new byte[1024];
-        int n;
-        while ((n = fis.read(b)) != -1) {
-            bos.write(b, 0, n);
-        }
-        fis.close();
-        byte[] data = bos.toByteArray();
-        bos.close();
-        return data;
-    }
-}

src/main/java/com/zdjizhi/utils/FlinkEnvironmentUtils.java

@@ -1,8 +1,6 @@
 package com.zdjizhi.utils;
 
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
-import org.apache.flink.streaming.api.CheckpointingMode;
-import org.apache.flink.streaming.api.environment.CheckpointConfig;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 
 
@@ -13,7 +11,7 @@ public class FlinkEnvironmentUtils {
     public static StreamExecutionEnvironment streamExeEnv = StreamExecutionEnvironment.getExecutionEnvironment();
 
     static {
-        streamExeEnv.setParallelism(CommonConfig.STREAM_EXECUTION_ENVIRONMENT_PARALLELISM);
+        streamExeEnv.setParallelism(FlowWriteConfig.STREAM_EXECUTION_ENVIRONMENT_PARALLELISM);
 
         /*
         // 每 1000ms 开始一次 checkpoint

src/main/java/com/zdjizhi/utils/HdfsUtils.java

@@ -1,75 +0,0 @@
-package com.zdjizhi.utils;
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.CommonConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-public class HdfsUtils {
-
-    private static final Log logger = LogFactory.get();
-    private static FileSystem fileSystem;
-
-    static {
-        Configuration configuration = new Configuration();
-        try {
-            //配置hdfs相关信息
-            configuration.set("fs.defaultFS","hdfs://ns1");
-            configuration.set("hadoop.proxyuser.root.hosts","*");
-            configuration.set("hadoop.proxyuser.root.groups","*");
-            configuration.set("ha.zookeeper.quorum", CommonConfig.HBASE_ZOOKEEPER_QUORUM);
-            configuration.set("dfs.nameservices","ns1");
-            configuration.set("dfs.ha.namenodes.ns1","nn1,nn2");
-            configuration.set("dfs.namenode.rpc-address.ns1.nn1",CommonConfig.HDFS_URI_NS1);
-            configuration.set("dfs.namenode.rpc-address.ns1.nn2",CommonConfig.HDFS_URI_NS2);
-            configuration.set("dfs.client.failover.proxy.provider.ns1","org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
-            //指定用户
-            System.setProperty("HADOOP_USER_NAME", CommonConfig.HDFS_USER);
-            //创建fileSystem,用于连接hdfs
-            fileSystem = FileSystem.get(configuration);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public static boolean isExists(String filePath) throws IOException {
-        return fileSystem.exists(new Path(filePath));
-    }
-
-    public static byte[] getFileBytes(String filePath) throws IOException {
-        try (FSDataInputStream open = fileSystem.open(new Path(filePath))) {
-            byte[] bytes = new byte[open.available()];
-            open.read(0, bytes, 0, open.available());
-            return bytes;
-        } catch (IOException e) {
-            logger.error("An I/O exception when files are download from HDFS. Message is :" + e.getMessage());
-        }
-        return null;
-    }
-
-    public static void uploadFileByBytes(String filePath,byte[] bytes) throws IOException {
-        try (FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path(filePath), true)) {
-            fsDataOutputStream.write(bytes);
-//            fsDataOutputStream.flush();
-        } catch (RuntimeException e) {
-            logger.error("Uploading files to the HDFS is abnormal. Message is :" + e.getMessage());
-        } catch (IOException e) {
-            logger.error("An I/O exception when files are uploaded to HDFS. Message is :" + e.getMessage());
-        }
-    }
-
-    public static void rename(String src, String dst) throws IOException {
-        fileSystem.rename(new Path(src),new Path(dst));
-    }
-
-
-}

src/main/java/com/zdjizhi/utils/HttpClientUtils.java

@@ -2,12 +2,16 @@ package com.zdjizhi.utils;
 
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.CommonConfig;
+import com.geedgenetworks.utils.StringUtil;
+
+import com.zdjizhi.common.FlowWriteConfig;
 import org.apache.http.*;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.*;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.conn.ConnectTimeoutException;
@@ -48,9 +52,9 @@ public class HttpClientUtils {
     static {
 
         // 设置最大连接数
-        CONN_MANAGER.setMaxTotal(CommonConfig.HTTP_POOL_MAX_CONNECTION);
+        CONN_MANAGER.setMaxTotal(FlowWriteConfig.HTTP_POOL_MAX_CONNECTION);
         // 设置每个连接的路由数
-        CONN_MANAGER.setDefaultMaxPerRoute(CommonConfig.HTTP_POOL_MAX_PER_ROUTE);
+        CONN_MANAGER.setDefaultMaxPerRoute(FlowWriteConfig.HTTP_POOL_MAX_PER_ROUTE);
 
     }
 
@@ -62,11 +66,11 @@ public class HttpClientUtils {
         // 创建Http请求配置参数
         RequestConfig requestConfig = RequestConfig.custom()
                 // 获取连接超时时间
-                .setConnectionRequestTimeout(CommonConfig.HTTP_POOL_REQUEST_TIMEOUT)
+                .setConnectionRequestTimeout(FlowWriteConfig.HTTP_POOL_REQUEST_TIMEOUT)
                 // 请求超时时间
-                .setConnectTimeout(CommonConfig.HTTP_POOL_CONNECT_TIMEOUT)
+                .setConnectTimeout(FlowWriteConfig.HTTP_POOL_CONNECT_TIMEOUT)
                 // 响应超时时间
-                .setSocketTimeout(CommonConfig.HTTP_POOL_RESPONSE_TIMEOUT)
+                .setSocketTimeout(FlowWriteConfig.HTTP_POOL_RESPONSE_TIMEOUT)
                 .build();
 
         /*

src/main/java/com/zdjizhi/utils/IpLocationConfiguration.java

@@ -1,21 +0,0 @@
-package com.zdjizhi.utils;
-import lombok.Data;
-/**
- * @author fy
- * @version 1.0
- * @date 2022/10/19 18:27
- */
-
-
-@Data
-public class IpLocationConfiguration {
-
-    private String ipV4UserDefined;
-
-    private String ipV4BuiltIn;
-
-    private String ipV6UserDefined;
-
-    private String ipV6BuiltIn;
-
-}

src/main/java/com/zdjizhi/utils/IpLookupUtils.java

@@ -0,0 +1,160 @@
+package com.zdjizhi.utils;
+
+import cn.hutool.crypto.digest.DigestUtil;
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.alibaba.fastjson2.*;
+import com.alibaba.nacos.api.config.ConfigService;
+import com.alibaba.nacos.api.config.listener.Listener;
+import com.alibaba.nacos.api.exception.NacosException;
+import com.geedgenetworks.utils.IpLookupV2;
+import com.geedgenetworks.utils.StringUtil;
+import com.google.common.base.Joiner;
+import com.zdjizhi.common.FlowWriteConfig;
+import com.zdjizhi.common.pojo.KnowlegeBaseMeta;
+import com.zdjizhi.utils.connections.http.HttpClientService;
+import com.zdjizhi.utils.connections.nacos.NacosConnection;
+
+import java.io.ByteArrayInputStream;
+import java.util.HashMap;
+import java.util.concurrent.Executor;
+
+/**
+ * @author wangchengcheng
+ * @version 2023/11/10 15:23
+ */
+public class IpLookupUtils {
+    private static final Log logger = LogFactory.get();
+    private static final String ipBuiltInName = "ip_builtin.mmdb";
+    private static final String ipUserDefinedName = "ip_user_defined.mmdb";
+
+    /**
+     * ip定位库
+     */
+    private static IpLookupV2 ipLookup;
+
+    /**
+     * 定位库默认分隔符
+     */
+    private static final String LOCATION_SEPARATOR = ".";
+
+    /**
+     * 最大重试次数
+     */
+    private static final int TRY_TIMES = 5;
+
+    /**
+     * http connections
+     */
+    private static final HttpClientService httpClientService;
+
+    /**
+     * 定位库元数据缓存
+     */
+    private static final HashMap<String, KnowlegeBaseMeta> knowledgeMetaCache = new HashMap<>(16);
+
+    static {
+        JSONPath jsonPath = JSONPath.of(getFilterParameter());
+        httpClientService = new HttpClientService();
+
+        NacosConnection nacosConnection = new NacosConnection();
+        ConfigService schemaService = nacosConnection.getPublicService();
+        try {
+            String configInfo = schemaService.getConfigAndSignListener(FlowWriteConfig.NACOS_KNOWLEDGEBASE_DATA_ID, FlowWriteConfig.NACOS_PUBLIC_GROUP, FlowWriteConfig.NACOS_CONNECTION_TIMEOUT, new Listener() {
+                @Override
+                public Executor getExecutor() {
+                    return null;
+                }
+
+                @Override
+                public void receiveConfigInfo(String configInfo) {
+                    if (StringUtil.isNotBlank(configInfo)) {
+                        updateIpLookup(jsonPath, configInfo);
+                    }
+                }
+            });
+
+            if (StringUtil.isNotBlank(configInfo)) {
+                updateIpLookup(jsonPath, configInfo);
+            }
+        } catch (NacosException e) {
+            logger.error("Get Schema config from Nacos error,The exception message is :" + e.getMessage());
+        }
+    }
+
+    private static void updateIpLookup(JSONPath jsonPath, String configInfo) {
+        String extract = jsonPath.extract(JSONReader.of(configInfo)).toString();
+        if (StringUtil.isNotBlank(extract)) {
+            JSONArray jsonArray = JSON.parseArray(extract);
+            if (jsonArray.size() > 0) {
+                for (int i = 0; i < jsonArray.size(); i++) {
+                    KnowlegeBaseMeta knowlegeBaseMeta = JSONObject.parseObject(jsonArray.getString(i), KnowlegeBaseMeta.class);
+                    String fileName = Joiner.on(LOCATION_SEPARATOR).useForNull("").join(knowlegeBaseMeta.getName(), knowlegeBaseMeta.getFormat());
+                    knowledgeMetaCache.put(fileName, knowlegeBaseMeta);
+                }
+                reloadIpLookup();
+            }
+        }
+    }
+
+    /**
+     * 从HDFS下载文件更新IpLookup
+     */
+    private static void reloadIpLookup() {
+        IpLookupV2.Builder builder = new IpLookupV2.Builder(false);
+        for (String fileName : knowledgeMetaCache.keySet()) {
+            int retryNum = 0;
+            KnowlegeBaseMeta knowlegeBaseMeta = knowledgeMetaCache.get(fileName);
+            String metaSha256 = knowlegeBaseMeta.getSha256();
+            while (retryNum < TRY_TIMES) {
+                System.out.println("download file ：" + fileName + ",HOS path :" + knowlegeBaseMeta.getPath());
+                Long startTime = System.currentTimeMillis();
+                byte[] httpGetByte = httpClientService.httpGetByte(knowlegeBaseMeta.getPath(), FlowWriteConfig.HTTP_SOCKET_TIMEOUT);
+                if (httpGetByte != null && httpGetByte.length > 0) {
+                    String downloadFileSha256 = DigestUtil.sha256Hex(httpGetByte);
+                    if (metaSha256.equals(downloadFileSha256)) {
+                        ByteArrayInputStream inputStream = new ByteArrayInputStream(httpGetByte);
+                        switch (fileName) {
+                            case ipBuiltInName:
+                                builder.loadDataFile(inputStream);
+                                break;
+                            case ipUserDefinedName:
+                                builder.loadDataFilePrivate(inputStream);
+                                break;
+                            default:
+                        }
+                        System.out.println("update " + fileName + " finished, speed :" + (System.currentTimeMillis() - startTime) + "ms");
+                        retryNum = TRY_TIMES;
+                    } else {
+                        logger.error("通过HOS下载{}的sha256为:{} ,Nacos内记录为:{} ,sha256不相等 开始第{}次重试下载文件", fileName, downloadFileSha256, metaSha256, retryNum);
+                        retryNum++;
+                    }
+                } else {
+                    logger.error("通过HOS下载{}的流为空 ,开始第{}次重试下载文件", fileName, retryNum);
+                    retryNum++;
+                }
+            }
+        }
+        ipLookup = builder.build();
+    }
+
+    /**
+     * 根据配置组合生成知识库元数据过滤参数
+     *
+     * @return 过滤参数
+     */
+    private static String getFilterParameter() {
+//       String expr = "$.[?(@.version=='latest' && @.name in ['ip_built_in','ip_user_defined'])].['name','sha256','format','path']";
+
+
+        String expr = "[?(@.version=='latest')][?(@.name in ('ip_builtin','ip_user_defined'))]";
+
+        return expr;
+    }
+
+    public static String getCountryLookup(String ip) {
+        return ipLookup.countryLookup(ip);
+    }
+
+
+}

src/main/java/com/zdjizhi/utils/IpUtils.java

@@ -1,224 +0,0 @@
-package com.zdjizhi.utils;
-
-import cn.hutool.core.io.file.FileReader;
-import cn.hutool.crypto.digest.DigestUtil;
-import com.zdjizhi.common.CommonConfig;
-import com.zdjizhi.common.CustomFile;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-
-public class IpUtils {
-    public static IpLookupV2 ipLookup ;
-
-    private static Logger LOG = LoggerFactory.getLogger(IpUtils.class);
-
-
-    /**
-     * IP定位库工具类
-     */
-//    public static IpLookupV2 ipLookup = new IpLookupV2.Builder(false)
-//            .loadDataFileV4(CommonConfig.IP_MMDB_PATH + "ip_v4_built_in.mmdb")
-//            .loadDataFileV6(CommonConfig.IP_MMDB_PATH + "ip_v6_built_in.mmdb")
-//            .loadDataFilePrivateV4(CommonConfig.IP_MMDB_PATH + "ip_v4_user_defined.mmdb")
-//            .loadDataFilePrivateV6(CommonConfig.IP_MMDB_PATH + "ip_v6_user_defined.mmdb")
-//            .build();
-
-
-    public static void loadIpLook(){
-        try {
-            IpLookupV2.Builder builder = new IpLookupV2.Builder(false);
-            if ("CLUSTER".equals(CommonConfig.CLUSTER_OR_SINGLE)) {
-                byte[] ipv4BuiltBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + "ip_v4_built_in.mmdb");
-                if (ipv4BuiltBytes!=null){
-                    InputStream ipv4BuiltInputStream = new ByteArrayInputStream(ipv4BuiltBytes);
-                    builder.loadDataFileV4(ipv4BuiltInputStream);
-                }
-
-                byte[] ipv6BuiltBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + "ip_v6_built_in.mmdb");
-                if (ipv6BuiltBytes!=null){
-                    InputStream ipv6BuiltInputStream = new ByteArrayInputStream(ipv6BuiltBytes);
-                    builder.loadDataFileV6(ipv6BuiltInputStream);
-                }
-
-                byte[] ipv4UserBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + "ip_v4_user_defined.mmdb");
-                if (ipv4UserBytes!=null){
-                    InputStream ipv4UserInputStream = new ByteArrayInputStream(ipv4UserBytes);
-                    builder.loadDataFilePrivateV4(ipv4UserInputStream);
-                }
-
-                byte[] ipv6UserBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + "ip_v6_user_defined.mmdb");
-                if (ipv6UserBytes!=null){
-                    InputStream ipv6UserInputStream = new ByteArrayInputStream(ipv6UserBytes);
-                    builder.loadDataFilePrivateV6(ipv6UserInputStream);
-                }
-            }else if ("SINGLE".equals(CommonConfig.CLUSTER_OR_SINGLE)){
-                byte[] ipv4BuiltBytes = FileByteUtils.getFileBytes(CommonConfig.DOWNLOAD_PATH + "ip_v4_built_in.mmdb");
-                if (ipv4BuiltBytes!=null){
-                    InputStream ipv4BuiltInputStream = new ByteArrayInputStream(ipv4BuiltBytes);
-                    builder.loadDataFileV4(ipv4BuiltInputStream);
-                }
-
-                byte[] ipv6BuiltBytes = FileByteUtils.getFileBytes(CommonConfig.DOWNLOAD_PATH + "ip_v6_built_in.mmdb");
-                if (ipv6BuiltBytes!=null){
-                    InputStream ipv6BuiltInputStream = new ByteArrayInputStream(ipv6BuiltBytes);
-                    builder.loadDataFileV6(ipv6BuiltInputStream);
-                }
-
-                byte[] ipv4UserBytes = FileByteUtils.getFileBytes(CommonConfig.DOWNLOAD_PATH + "ip_v4_user_defined.mmdb");
-                if (ipv4UserBytes!=null){
-                    InputStream ipv4UserInputStream = new ByteArrayInputStream(ipv4UserBytes);
-                    builder.loadDataFilePrivateV4(ipv4UserInputStream);
-                }
-
-                byte[] ipv6UserBytes = FileByteUtils.getFileBytes(CommonConfig.DOWNLOAD_PATH + "ip_v6_user_defined.mmdb");
-                if (ipv6UserBytes!=null){
-                    InputStream ipv6UserInputStream = new ByteArrayInputStream(ipv6UserBytes);
-                    builder.loadDataFilePrivateV6(ipv6UserInputStream);
-                }
-            }
-            ipLookup = builder.build();
-
-        }catch (Exception e){
-            LOG.error("加载失败",e);
-        }
-    }
-
-    public static void updateIpLook(Map<String, String> knowledgeFileCache){
-        try{
-            IpLookupV2.Builder builder = new IpLookupV2.Builder(false);
-            if ("CLUSTER".equals(CommonConfig.CLUSTER_OR_SINGLE)) {
-                byte[] ipv4BuiltBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + "ip_v4_built_in.mmdb");
-                if (ipv4BuiltBytes!=null){
-                    if (knowledgeFileCache.containsKey("ip_v4_built_in.mmdb")){
-                        String sha256 = knowledgeFileCache.get("ip_v4_built_in.mmdb");
-                        byte[] localFileByte = getLocalFile("ip_v4_built_in.mmdb");
-                        String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                        if (sha256.equals(localFileSha256Hex)){
-                            builder.loadDataFileV4(new ByteArrayInputStream(ipv4BuiltBytes));
-                        }
-                    }
-                }
-
-                byte[] ipv6BuiltBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + "ip_v6_built_in.mmdb");
-                if (ipv6BuiltBytes!=null){
-                    if (knowledgeFileCache.containsKey("ip_v6_built_in.mmdb")) {
-                        String sha256 = knowledgeFileCache.get("ip_v6_built_in.mmdb");
-                        byte[] localFileByte = getLocalFile("ip_v6_built_in.mmdb");
-                        String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                        if (sha256.equals(localFileSha256Hex)) {
-                            builder.loadDataFileV6(new ByteArrayInputStream(ipv6BuiltBytes));
-                        }
-                    }
-                }
-
-                byte[] ipv4UserBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + "ip_v4_user_defined.mmdb");
-                if (ipv4UserBytes!=null){
-                    if (knowledgeFileCache.containsKey("ip_v4_user_defined.mmdb")) {
-                        String sha256 = knowledgeFileCache.get("ip_v4_user_defined.mmdb");
-                        byte[] localFileByte = getLocalFile("ip_v4_user_defined.mmdb");
-                        String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                        if (sha256.equals(localFileSha256Hex)) {
-                            builder.loadDataFilePrivateV4(new ByteArrayInputStream(ipv4UserBytes));
-                        }
-                    }
-                }
-
-                byte[] ipv6UserBytes = HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + "ip_v6_user_defined.mmdb");
-                if (ipv6UserBytes!=null){
-                    if (knowledgeFileCache.containsKey("ip_v6_user_defined.mmdb")) {
-                        String sha256 = knowledgeFileCache.get("ip_v6_user_defined.mmdb");
-                        byte[] localFileByte = getLocalFile("ip_v6_user_defined.mmdb");
-                        String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                        if (sha256.equals(localFileSha256Hex)) {
-                            builder.loadDataFilePrivateV6(new ByteArrayInputStream(ipv6UserBytes));
-                        }
-                    }
-                }
-            }else if ("SINGLE".equals(CommonConfig.CLUSTER_OR_SINGLE)){
-                byte[] ipv4BuiltBytes = FileByteUtils.getFileBytes(CommonConfig.DOWNLOAD_PATH + "ip_v4_built_in.mmdb");
-                if (ipv4BuiltBytes!=null){
-                    if (knowledgeFileCache.containsKey("ip_v4_built_in.mmdb")){
-                        String sha256 = knowledgeFileCache.get("ip_v4_built_in.mmdb");
-                        byte[] localFileByte = getLocalFile("ip_v4_built_in.mmdb");
-                        String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                        if (sha256.equals(localFileSha256Hex)){
-                            builder.loadDataFileV4(new ByteArrayInputStream(ipv4BuiltBytes));
-                        }
-                    }
-                }
-
-                byte[] ipv6BuiltBytes = FileByteUtils.getFileBytes(CommonConfig.DOWNLOAD_PATH + "ip_v6_built_in.mmdb");
-                if (ipv6BuiltBytes!=null){
-                    if (knowledgeFileCache.containsKey("ip_v6_built_in.mmdb")) {
-                        String sha256 = knowledgeFileCache.get("ip_v6_built_in.mmdb");
-                        byte[] localFileByte = getLocalFile("ip_v6_built_in.mmdb");
-                        String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                        if (sha256.equals(localFileSha256Hex)) {
-                            builder.loadDataFileV6(new ByteArrayInputStream(ipv6BuiltBytes));
-                        }
-                    }
-                }
-
-                byte[] ipv4UserBytes = FileByteUtils.getFileBytes(CommonConfig.DOWNLOAD_PATH + "ip_v4_user_defined.mmdb");
-                if (ipv4UserBytes!=null){
-                    if (knowledgeFileCache.containsKey("ip_v4_user_defined.mmdb")) {
-                        String sha256 = knowledgeFileCache.get("ip_v4_user_defined.mmdb");
-                        byte[] localFileByte = getLocalFile("ip_v4_user_defined.mmdb");
-                        String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                        if (sha256.equals(localFileSha256Hex)) {
-                            builder.loadDataFilePrivateV4(new ByteArrayInputStream(ipv4UserBytes));
-                        }
-                    }
-                }
-
-                byte[] ipv6UserBytes = FileByteUtils.getFileBytes(CommonConfig.DOWNLOAD_PATH + "ip_v6_user_defined.mmdb");
-                if (ipv6UserBytes!=null){
-                    if (knowledgeFileCache.containsKey("ip_v6_user_defined.mmdb")) {
-                        String sha256 = knowledgeFileCache.get("ip_v6_user_defined.mmdb");
-                        byte[] localFileByte = getLocalFile("ip_v6_user_defined.mmdb");
-                        String localFileSha256Hex = DigestUtil.sha256Hex(localFileByte);
-                        if (sha256.equals(localFileSha256Hex)) {
-                            builder.loadDataFilePrivateV6(new ByteArrayInputStream(ipv6UserBytes));
-                        }
-                    }
-                }
-            }
-            ipLookup = builder.build();
-        }catch (Exception e){
-            LOG.error("加载失败",e);
-        }
-
-    }
-
-
-    private static byte[] getLocalFile(String name) {
-        byte[] fileBytes = null;
-        try {
-            fileBytes = "CLUSTER".equals(CommonConfig.CLUSTER_OR_SINGLE) ?
-                    HdfsUtils.getFileBytes(CommonConfig.HDFS_PATH + name) :
-                    new FileReader(CommonConfig.DOWNLOAD_PATH + name).readBytes();
-        } catch (RuntimeException | IOException e) {
-            e.printStackTrace();
-        }
-        return fileBytes;
-    }
-
-    public static void main(String[] args) {
-        System.out.println(ipLookup.countryLookup("49.7.115.37"));
-
-        String ips = "182.168.50.23,182.168.50.45,182.168.56.9,182.168.56.8,92.168.50.58,19.168.56.7,12.168.56.6,2.168.50.40,1.168.50.19,9.168.50.6,2.168.50.4,192.168.56.17,192.168.50.27,192.168.50.26,192.168.50.18,192.168.56.3,192.168.56.10";
-        for (String ip:ips.split(",")){
-            System.out.println(ip+"--"+ipLookup.countryLookup(ip));
-        }
-
-    }
-
-
-
-}

src/main/java/com/zdjizhi/utils/KafkaUtils.java

@@ -1,6 +1,6 @@
 package com.zdjizhi.utils;
 
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 import org.apache.flink.api.common.serialization.SimpleStringSchema;
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
 
@@ -11,11 +11,11 @@ public class KafkaUtils {
 
     private static Properties getKafkaSinkProperty(){
         Properties properties = new Properties();
-        properties.setProperty("bootstrap.servers", CommonConfig.KAFKA_OUTPUT_BOOTSTRAP_SERVERS);
+        properties.setProperty("bootstrap.servers", FlowWriteConfig.KAFKA_OUTPUT_BOOTSTRAP_SERVERS);
-        if (CommonConfig.SASL_JAAS_CONFIG_FLAG == 1){
+        if (FlowWriteConfig.SASL_JAAS_CONFIG_FLAG == 1){
             properties.put("security.protocol", "SASL_PLAINTEXT");
             properties.put("sasl.mechanism", "PLAIN");
-            properties.put("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\""+CommonConfig.SASL_JAAS_CONFIG_USER+"\" password=\""+CommonConfig.SASL_JAAS_CONFIG_PASSWORD+"\";");
+            properties.put("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\""+ FlowWriteConfig.SASL_JAAS_CONFIG_USER+"\" password=\""+ FlowWriteConfig.SASL_JAAS_CONFIG_PASSWORD+"\";");
         }
 
         return properties;

src/main/java/com/zdjizhi/utils/SnowflakeId.java

@@ -2,7 +2,7 @@ package com.zdjizhi.utils;
 
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
 
 public class SnowflakeId {
 //    private static final Logger logger = LoggerFactory.getLogger(SnowflakeId.class);
@@ -99,7 +99,7 @@ public class SnowflakeId {
     private static ZookeeperUtils zookeeperUtils = new ZookeeperUtils();
 
     static {
-        idWorker = new SnowflakeId(CommonConfig.HBASE_ZOOKEEPER_QUORUM, CommonConfig.DATA_CENTER_ID_NUM);
+        idWorker = new SnowflakeId(FlowWriteConfig.HBASE_ZOOKEEPER_QUORUM, FlowWriteConfig.DATA_CENTER_ID_NUM);
     }
 
     //==============================Constructors=====================================
@@ -108,7 +108,7 @@ public class SnowflakeId {
      * 构造函数
      */
     private SnowflakeId(String zookeeperIp, long dataCenterIdNum) {
-        DistributedLock lock = new DistributedLock(CommonConfig.HBASE_ZOOKEEPER_QUORUM, "disLocks1");
+        DistributedLock lock = new DistributedLock(FlowWriteConfig.HBASE_ZOOKEEPER_QUORUM, "disLocks1");
         try {
             lock.lock();
             int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + "worker" + dataCenterIdNum, zookeeperIp);

src/main/java/com/zdjizhi/utils/connections/http/HttpClientService.java

@@ -1,6 +1,11 @@
-package com.zdjizhi.utils;
+package com.zdjizhi.utils.connections.http;
 
-import com.zdjizhi.common.CommonConfig;
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+
+import com.geedgenetworks.utils.StringUtil;
+import com.zdjizhi.common.FlowWriteConfig;
+import com.zdjizhi.utils.exception.FlowWriteException;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.*;
 import org.apache.http.client.ClientProtocolException;
@@ -8,63 +13,35 @@ import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.conn.ConnectionKeepAliveStrategy;
-import org.apache.http.conn.HttpHostConnectException;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.message.BasicHeaderElementIterator;
 import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.*;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InterruptedIOException;
-import java.net.URI;
+import java.net.SocketTimeoutException;
 import java.net.UnknownHostException;
-import java.nio.charset.StandardCharsets;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.X509Certificate;
-import java.util.Map;
 
-import static org.apache.kafka.common.requests.FetchMetadata.log;
+public class HttpClientService {
 
-/**
+    private static final Log log = LogFactory.get();
- * http client工具类
- */
-public class HttpClientUtils2 {
-    /** 全局连接池对象 */
-    private static final PoolingHttpClientConnectionManager CONN_MANAGER = new PoolingHttpClientConnectionManager();
-
-    private static Logger logger = LoggerFactory.getLogger(HttpClientUtils2.class);
-    public static final String ERROR_MESSAGE = "-1";
-
-    /*
-     * 静态代码块配置连接池信息
-     */
-    static {
-
-        // 设置最大连接数
-        CONN_MANAGER.setMaxTotal(CommonConfig.HTTP_POOL_MAX_CONNECTION);
-        // 设置每个连接的路由数
-        CONN_MANAGER.setDefaultMaxPerRoute(CommonConfig.HTTP_POOL_MAX_PER_ROUTE);
-
-    }
 
     /**
      * 在调用SSL之前需要重写验证方法，取消检测SSL
@@ -80,9 +57,11 @@ public class HttpClientUtils2 {
                 public X509Certificate[] getAcceptedIssuers() {
                     return null;
                 }
+
                 @Override
                 public void checkClientTrusted(X509Certificate[] xcs, String str) {
                 }
+
                 @Override
                 public void checkServerTrusted(X509Certificate[] xcs, String str) {
                 }
@@ -96,31 +75,33 @@ public class HttpClientUtils2 {
             // 创建ConnectionManager，添加Connection配置信息
             PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
             // 设置最大连接数
-            connManager.setMaxTotal(CommonConfig.HTTP_POOL_MAX_CONNECTION);
+            connManager.setMaxTotal(FlowWriteConfig.HTTP_POOL_MAX_CONNECTION);
             // 设置每个连接的路由数
-            connManager.setDefaultMaxPerRoute(CommonConfig.HTTP_POOL_MAX_PER_ROUTE);
+            connManager.setDefaultMaxPerRoute(FlowWriteConfig.HTTP_POOL_MAX_PER_ROUTE);
             return connManager;
         } catch (KeyManagementException | NoSuchAlgorithmException e) {
-            throw new RuntimeException(e.getMessage());
+            throw new FlowWriteException(e.getMessage());
         }
     }
 
     /**
      * 获取Http客户端连接对象
+     *
+     * @param socketTimeOut 响应超时时间
      * @return Http客户端连接对象
      */
-    private  CloseableHttpClient getHttpClient() {
+    private CloseableHttpClient getHttpClient(int socketTimeOut) {
         // 创建Http请求配置参数
         RequestConfig requestConfig = RequestConfig.custom()
                 // 获取连接超时时间
-                .setConnectionRequestTimeout(CommonConfig.HTTP_POOL_REQUEST_TIMEOUT)
+                .setConnectionRequestTimeout(FlowWriteConfig.HTTP_POOL_REQUEST_TIMEOUT)
                 // 请求超时时间
-                .setConnectTimeout(CommonConfig.HTTP_POOL_CONNECT_TIMEOUT)
+                .setConnectTimeout(FlowWriteConfig.HTTP_POOL_CONNECT_TIMEOUT)
                 // 响应超时时间
-                .setSocketTimeout(CommonConfig.HTTP_POOL_RESPONSE_TIMEOUT)
+                .setSocketTimeout(socketTimeOut)
                 .build();
 
-        /*
+        /**
          * 测出超时重试机制为了防止超时不生效而设置
          *  如果直接放回false,不重试
          *  这里会根据情况进行判断是否重试
@@ -135,25 +116,32 @@ public class HttpClientUtils2 {
             if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
                 return false;
             }
+            if (exception instanceof SocketTimeoutException) {
+                if (exception.getMessage().contains("Read timed out")) {
+                    return false;
+                }
+            }
             if (exception instanceof UnknownHostException) {// 目标服务器不可达
                 return false;
             }
             if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
                 return false;
             }
-            if (exception instanceof HttpHostConnectException) {// 连接被拒绝
-                return false;
-            }
             if (exception instanceof SSLException) {// ssl握手异常
                 return false;
             }
             if (exception instanceof InterruptedIOException) {// 超时
                 return true;
             }
+
+
             HttpClientContext clientContext = HttpClientContext.adapt(context);
             HttpRequest request = clientContext.getRequest();
             // 如果请求是幂等的，就再次尝试
-            return !(request instanceof HttpEntityEnclosingRequest);
+            if (!(request instanceof HttpEntityEnclosingRequest)) {
+                return true;
+            }
+            return false;
         };
 
 
@@ -164,7 +152,7 @@ public class HttpClientUtils2 {
                 HeaderElement he = it.nextElement();
                 String param = he.getName();
                 String value = he.getValue();
-                if (value != null && "timeout".equalsIgnoreCase(param)) {
+                if (value != null && param.equalsIgnoreCase("timeout")) {
                     return Long.parseLong(value) * 1000;
                 }
             }
@@ -183,11 +171,10 @@ public class HttpClientUtils2 {
                 .build();
     }
 
-    // TODO: 2022/10/19  加载知识库
     public InputStream httpGetInputStream(String url, int socketTimeout, Header... headers) {
         InputStream result = null;
         // 获取客户端连接对象
-        CloseableHttpClient httpClient = getHttpClient();// TODO: 2022/10/19 去掉了 socketTimeout
+        CloseableHttpClient httpClient = getHttpClient(socketTimeout);
         // 创建GET请求对象
         HttpGet httpGet = new HttpGet(url);
         if (StringUtil.isNotEmpty(headers)) {
@@ -228,7 +215,47 @@ public class HttpClientUtils2 {
     }
 
 
+    public byte[] httpGetByte(String url, int socketTimeout, Header... headers) {
+        byte[] result = null;
+        // 获取客户端连接对象
+        CloseableHttpClient httpClient = getHttpClient(socketTimeout);
+        // 创建GET请求对象
+        HttpGet httpGet = new HttpGet(url);
+        if (StringUtil.isNotEmpty(headers)) {
+            for (Header h : headers) {
+                httpGet.addHeader(h);
+            }
+        }
+        CloseableHttpResponse response = null;
 
+        try {
+            // 执行请求
+            response = httpClient.execute(httpGet);
+            // 获取响应实体
+            result = IOUtils.toByteArray(response.getEntity().getContent());
+            // 获取响应信息
+            EntityUtils.consume(response.getEntity());
+        } catch (ClientProtocolException e) {
+            log.error("current file: {},Protocol error:{}", url, e.getMessage());
 
+        } catch (ParseException e) {
+            log.error("current file: {}, Parser error:{}", url, e.getMessage());
+
+        } catch (IOException e) {
+            log.error("current file: {},IO error:{}", url, e.getMessage());
+
+        } finally {
+            if (null != response) {
+                try {
+                    EntityUtils.consume(response.getEntity());
+                    response.close();
+                } catch (IOException e) {
+                    log.error("Release Connection error:{}", e.getMessage());
+
+                }
+            }
+            return result;
+        }
+    }
 
 }

src/main/java/com/zdjizhi/utils/connections/nacos/NacosConnection.java

@@ -0,0 +1,54 @@
+package com.zdjizhi.utils.connections.nacos;
+
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.alibaba.nacos.api.NacosFactory;
+import com.alibaba.nacos.api.PropertyKeyConst;
+import com.alibaba.nacos.api.config.ConfigService;
+import com.alibaba.nacos.api.exception.NacosException;
+import com.zdjizhi.common.FlowWriteConfig;
+
+
+import java.util.Properties;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.tools.connections.nacos
+ * @Description:
+ * @date 2023/7/2714:49
+ */
+public class NacosConnection {
+    private static final Log logger = LogFactory.get();
+
+    private ConfigService configService;
+
+
+    public ConfigService getDosService() {
+        Properties properties = new Properties();
+        properties.setProperty(PropertyKeyConst.SERVER_ADDR, FlowWriteConfig.NACOS_SERVER);
+        properties.setProperty(PropertyKeyConst.NAMESPACE, FlowWriteConfig.NACOS_DOS_NAMESPACE);
+        properties.setProperty(PropertyKeyConst.USERNAME, FlowWriteConfig.NACOS_USERNAME);
+        properties.setProperty(PropertyKeyConst.PASSWORD, FlowWriteConfig.NACOS_PIN);
+        try {
+            configService = NacosFactory.createConfigService(properties);
+        } catch (NacosException e) {
+            logger.error("NacosException:{}", e);
+        }
+        return configService;
+    }
+
+
+    public ConfigService getPublicService() {
+        Properties properties = new Properties();
+        properties.setProperty(PropertyKeyConst.SERVER_ADDR, FlowWriteConfig.NACOS_SERVER);
+        properties.setProperty(PropertyKeyConst.NAMESPACE, FlowWriteConfig.NACOS_PUBLIC_NAMESPACE);
+        properties.setProperty(PropertyKeyConst.USERNAME, FlowWriteConfig.NACOS_USERNAME);
+        properties.setProperty(PropertyKeyConst.PASSWORD, FlowWriteConfig.NACOS_PIN);
+        try {
+            configService = NacosFactory.createConfigService(properties);
+        } catch (NacosException e) {
+            logger.error("NacosException:{}", e);
+        }
+        return configService;
+    }
+}

src/main/java/com/zdjizhi/utils/connections/nacos/NacosUtils.java

@@ -1,11 +1,10 @@
-package com.zdjizhi.utils;
+package com.zdjizhi.utils.connections.nacos;
 
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
-import com.alibaba.nacos.api.NacosFactory;
-import com.alibaba.nacos.api.PropertyKeyConst;
 import com.alibaba.nacos.api.config.ConfigService;
 import com.alibaba.nacos.api.config.listener.Listener;
+import com.zdjizhi.common.FlowWriteConfig;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -13,40 +12,18 @@ import java.util.Properties;
 import java.util.concurrent.Executor;
 
 public class NacosUtils {
-//    private static final Logger logger = LoggerFactory.getLogger(NacosUtils.class);
     private static final Log logger = LogFactory.get();
-    private static Properties nacosProperties = new Properties();
     private static Properties commonProperties = new Properties();
 
-
-    private static final String NACOS_SERVER_ADDR = CommonConfigurations.getStringProperty("nacos.server.addr");
-    private static final String NACOS_STATIC_NAMESPACE = CommonConfigurations.getStringProperty("nacos.static.namespace");
-    private static final String NACOS_USERNAME = CommonConfigurations.getStringProperty("nacos.username");
-    private static final String NACOS_PASSWORD = CommonConfigurations.getStringProperty("nacos.password");
-    private static final String NACOS_STATIC_DATA_ID = CommonConfigurations.getStringProperty("nacos.static.data.id");
-    private static final String NACOS_STATIC_GROUP = CommonConfigurations.getStringProperty("nacos.static.group");
-    private static final long NACOS_READ_TIMEOUT = CommonConfigurations.getLongProperty("nacos.read.timeout");
-
     static {
-        createConfigService();
+        NacosConnection nacosConnection = new NacosConnection();
-    }
+        ConfigService dosService = nacosConnection.getDosService();
-
-    private static void getProperties() {
-        nacosProperties.setProperty(PropertyKeyConst.SERVER_ADDR, NACOS_SERVER_ADDR);
-        nacosProperties.setProperty(PropertyKeyConst.NAMESPACE, NACOS_STATIC_NAMESPACE);
-        nacosProperties.setProperty(PropertyKeyConst.USERNAME, NACOS_USERNAME);
-        nacosProperties.setProperty(PropertyKeyConst.PASSWORD, NACOS_PASSWORD);
-    }
-
-    private static void createConfigService() {
         try {
-            getProperties();
+            String config = dosService.getConfig(FlowWriteConfig.NACOS_DOS_DATA_ID, FlowWriteConfig.NACOS_DOS_GROUP, FlowWriteConfig.NACOS_CONNECTION_TIMEOUT);
-            ConfigService configService = NacosFactory.createConfigService(nacosProperties);
+
-            String config = configService.getConfig(NACOS_STATIC_DATA_ID, NACOS_STATIC_GROUP, NACOS_READ_TIMEOUT);
             commonProperties.load(new StringReader(config));
 
-
+            dosService.addListener(FlowWriteConfig.NACOS_DOS_DATA_ID, FlowWriteConfig.NACOS_DOS_GROUP, new Listener() {
-            configService.addListener(NACOS_STATIC_DATA_ID, NACOS_STATIC_GROUP, new Listener() {
                 @Override
                 public Executor getExecutor() {
                     return null;

src/main/java/com/zdjizhi/utils/exception/FlowWriteException.java

@@ -0,0 +1,13 @@
+package com.zdjizhi.utils.exception;
+
+
+public class FlowWriteException extends RuntimeException {
+
+    public FlowWriteException() {
+    }
+
+    public FlowWriteException(String message) {
+        super(message);
+    }
+
+}

src/main/resources/common.properties

@@ -74,10 +74,6 @@ destination.ip.partition.num=10000
 
 data.center.id.num=15
 
-#IP mmdb库路径
-ip.mmdb.path=D:\\data\\dat\\bak\\
-#ip.mmdb.path=/home/bigdata/topology/dat/
-#ip.mmdb.path=/home/bigdata/wlh/topology/dos-detection/dat/
 
 #bifang服务访问地址
 bifang.server.uri=http://192.168.44.72:80
@@ -90,10 +86,10 @@ bifang.server.encryptpwd.path=/v1/user/encryptpwd
 bifang.server.login.path=/v1/user/login
 
 #获取vaysId路径信息
-bifang.server.policy.vaysid.path=/v1/system/vsys/
+bifang.server.policy.vaysid.path=/v1/admin/vsys
 
 #获取静态阈值路径信息
-bifang.server.policy.threshold.path=/v1/policy/profile/DoS/detection/threshold
+bifang.server.policy.threshold.path=/v1/policy/profile/dos/detection/threshold
 
 #http请求相关参数
 #最大连接数
@@ -129,30 +125,15 @@ sasl.jaas.config.flag=1
 nacos.server.addr=192.168.44.12:8848
 nacos.username=nacos
 nacos.password=nacos
-nacos.read.timeout=5000
 ############################## Nacos ---知识库配置 ######################################
 nacos.namespace=public
 nacos.data.id=knowledge_base.json
 nacos.group=DEFAULT_GROUP
-
+nacos.connection.timeout=60000
 
 ############################## Nacos ---静态阈值配置 ######################################
-nacos.static.namespace=test
+nacos.dos.namespace=test
-nacos.static.data.id=dos_detection.properties
+nacos.dos.data.id=dos_detection.properties
-nacos.static.group=Galaxy
+nacos.dos.group=Galaxy
 
-############################## hos Token 配置 ######################################
+http.socket.timeout=90000
-hos.token=c21f969b5f03d33d43e04f8f136e7682
-
-############################# 选择集群模式或者单机模式 配置 ######################################
-cluster.or.single=CLUSTER
-#cluster.or.single=SINGLE
-
-############################## 集群模式配置文件路径 配置 ######################################
-hdfs.path=/test/TEST/
-hdfs.uri.nn1=192.168.40.151:9000
-hdfs.uri.nn2=192.168.40.152:9000
-hdfs.user=dos
-
-############################## 单机模式配置文件下载路径 配置 ######################################
-download.path=D:\\ttt\\

src/main/resources/core-site.xml

@@ -1,58 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-    <property>  
-        <name>fs.defaultFS</name>  
-        <value>hdfs://ns1</value>  
-    </property>
-    <property>
-        <name>hadoop.tmp.dir</name>
-        <value>file:/home/tsg/olap/hadoop/tmp</value>
-    </property>
-    <property>
-        <name>io.file.buffer.size</name>
-        <value>131702</value>
-    </property>
-    <property>
-	<name>hadoop.proxyuser.root.hosts</name>
-	<value>*</value>
-    </property>
-    <property>
-	<name>hadoop.proxyuser.root.groups</name>
-	<value>*</value>
-    </property>
-    <property>
-	<name>hadoop.logfile.size</name>
-	<value>10000000</value>
-	<description>The max size of each log file</description>
-    </property>
-    <property>
-	<name>hadoop.logfile.count</name>
-	<value>1</value>
-	<description>The max number of log files</description>
-    </property>
-    <property>  
-        <name>ha.zookeeper.quorum</name>  
-        <value>192.168.40.151:2181,192.168.40.152:2181,192.168.40.203:2181</value>  
-    </property>
-    <property>
-        <name>ipc.client.connect.timeout</name>
-        <value>90000</value>
-    </property>
-</configuration>

src/main/resources/hdfs-site.xml

@@ -1,142 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-    <property>
-        <name>dfs.namenode.name.dir</name>
-        <value>file:/home/tsg/olap/hadoop/dfs/name</value>
-    </property>
-    <property>
-        <name>dfs.datanode.data.dir</name>
-        <value>file:/home/tsg/olap/hadoop/dfs/data</value>
-    </property>
-    <property>
-        <name>dfs.replication</name>
-        <value>2</value>
-    </property>
-    <property>
-	<name>dfs.webhdfs.enabled</name>
-	<value>true</value>
-    </property>
-    <property>
-        <name>dfs.permissions</name>
-        <value>false</value>
-    </property>	
-	<property>  
-        <name>dfs.permissions.enabled</name>  
-        <value>false</value>  
-    </property>  
-    <property>  
-        <name>dfs.nameservices</name>  
-        <value>ns1</value>  
-    </property>  
-    <property>  
-        <name>dfs.blocksize</name>  
-        <value>134217728</value>  
-    </property>  
-    <property>  
-        <name>dfs.ha.namenodes.ns1</name>  
-        <value>nn1,nn2</value>  
-    </property>  
-    <!-- nn1的RPC通信地址，nn1所在地址  -->  
-    <property>  
-        <name>dfs.namenode.rpc-address.ns1.nn1</name>  
-        <value>192.168.40.151:9000</value>  
-    </property>  
-    <!-- nn1的http通信地址，外部访问地址 -->  
-    <property>  
-        <name>dfs.namenode.http-address.ns1.nn1</name>  
-        <value>192.168.40.151:50070</value>  
-    </property>  
-    <!-- nn2的RPC通信地址，nn2所在地址 -->  
-    <property>  
-        <name>dfs.namenode.rpc-address.ns1.nn2</name>  
-        <value>192.168.40.152:9000</value>  
-    </property>  
-    <!-- nn2的http通信地址，外部访问地址 -->  
-    <property>  
-        <name>dfs.namenode.http-address.ns1.nn2</name>  
-        <value>192.168.40.152:50070</value>  
-    </property>  
-    <!-- 指定NameNode的元数据在JournalNode日志上的存放位置(一般和zookeeper部署在一起) -->  
-    <property>  
-        <name>dfs.namenode.shared.edits.dir</name>  
-        <value>qjournal://192.168.40.151:8485;192.168.40.152:8485;192.168.40.203:8485/ns1</value>  
-    </property>  
-    <!-- 指定JournalNode在本地磁盘存放数据的位置 -->  
-    <property>  
-        <name>dfs.journalnode.edits.dir</name>  
-        <value>/home/tsg/olap/hadoop/journal</value>  
-    </property>  
-    <!--客户端通过代理访问namenode，访问文件系统，HDFS 客户端与Active 节点通信的Java 类，使用其确定Active 节点是否活跃  -->  
-    <property>  
-        <name>dfs.client.failover.proxy.provider.ns1</name>  
-        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>  
-    </property>  
-    <!--这是配置自动切换的方法，有多种使用方法，具体可以看官网，在文末会给地址，这里是远程登录杀死的方法  -->  
-    <property>  
-        <name>dfs.ha.fencing.methods</name>  
-        <value>sshfence</value>    
-        <value>shell(true)</value> 
-    </property>  
-    <!-- 这个是使用sshfence隔离机制时才需要配置ssh免登陆 -->  
-    <property>  
-        <name>dfs.ha.fencing.ssh.private-key-files</name>  
-        <value>/root/.ssh/id_rsa</value>  
-    </property>  
-    <!-- 配置sshfence隔离机制超时时间，这个属性同上，如果你是用脚本的方法切换，这个应该是可以不配置的 -->  
-    <property>  
-        <name>dfs.ha.fencing.ssh.connect-timeout</name>  
-        <value>30000</value>  
-    </property>  
-    <!-- 这个是开启自动故障转移，如果你没有自动故障转移，这个可以先不配 -->  
-    <property>  
-        <name>dfs.ha.automatic-failover.enabled</name>  
-        <value>true</value>  
-    </property>
-    <property>
-        <name>dfs.datanode.max.transfer.threads</name>
-        <value>8192</value>
-    </property>
-    <!-- namenode处理RPC请求线程数，增大该值资源占用不大 -->
-    <property>
-        <name>dfs.namenode.handler.count</name>
-        <value>30</value>
-    </property>
-    <!-- datanode处理RPC请求线程数，增大该值会占用更多内存 -->
-    <property>
-        <name>dfs.datanode.handler.count</name>
-        <value>40</value>
-    </property>
-    <!-- balance时可占用的带宽 -->
-    <property>
-        <name>dfs.balance.bandwidthPerSec</name>
-        <value>104857600</value>
-    </property>
-    <!-- 磁盘预留空间，该空间不会被hdfs占用，单位字节-->
-    <property>
-        <name>dfs.datanode.du.reserved</name>
-        <value>5368709120</value>
-    </property>
-    <!-- datanode与namenode连接超时时间，单位毫秒  2 * heartbeat.recheck.interval + 30000  -->
-    <property>
-	<name>heartbeat.recheck.interval</name>
-	<value>100000</value>
-    </property>
-</configuration>
-

src/main/resources/yarn-site.xml

@@ -1,196 +0,0 @@
-<?xml version="1.0"?>
-<!--
-            Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-<configuration>
-    <property>
-        <name>yarn.nodemanager.aux-services</name>
-        <value>mapreduce_shuffle</value>
-    </property>
-        <property>
-       <name>yarn.resourcemanager.ha.enabled</name>
-       <value>true</value>
-    </property>
-    <!--声明两台resourcemanager的地址-->
-    <property>
-       <name>yarn.resourcemanager.cluster-id</name>
-       <value>rmcluster</value>
-    </property>
-    <property>
-       <name>yarn.resourcemanager.ha.rm-ids</name>
-       <value>rsm1,rsm2</value>
-    </property>
-    <!-- 配置rm1-->
-     <property>
-       <name>yarn.resourcemanager.hostname.rsm1</name>
-       <value>192.168.40.152</value>
-    </property>
-    <property>
-       <!--<name>yarn.resourcemanager.hostname.rm1</name>-->
-       <name>yarn.resourcemanager.address.rsm1</name>
-       <value>192.168.40.152:9916</value>
-    </property>
-    <property>
-           <name>yarn.resourcemanager.scheduler.address.rsm1</name>
-       <value>192.168.40.152:9917</value>
-    </property>
-    <property>
-           <name>yarn.resourcemanager.webapp.address.rsm1</name>
-       <value>192.168.40.152:9918</value>
-    </property>
-     <property>
-       <name>yarn.resourcemanager.admin.address.rsm1</name>
-       <value>192.168.40.152:9919</value>
-    </property>
-    <property>
-       <name>yarn.resourcemanager.resource-tracker.address.rsm1</name>
-       <value>192.168.40.152:9920</value>
-    </property>
-    <property>
-  <name>yarn.resourcemanager.ha.admin.address.rsm1</name>
-  <value>192.168.40.152:23142</value>
-    </property>
-    <!-- 配置rm2-->
-   
-    <property>
-       <name>yarn.resourcemanager.hostname.rsm2</name>
-       <value>192.168.40.203</value>
-    </property>
-    <property>
-       <!--<name>yarn.resourcemanager.hostname.rm1</name>-->
-       <name>yarn.resourcemanager.address.rsm2</name>
-       <value>192.168.40.203:9916</value>
-    </property>
-    <property>
-           <name>yarn.resourcemanager.scheduler.address.rsm2</name>
-       <value>192.168.40.203:9917</value>
-    </property>
-    <property>
-           <name>yarn.resourcemanager.webapp.address.rsm2</name>
-       <value>192.168.40.203:9918</value>
-    </property>
-    <property>
-       <name>yarn.resourcemanager.admin.address.rsm2</name>
-       <value>192.168.40.203:9919</value>
-    </property>
-    <property>
-       <name>yarn.resourcemanager.resource-tracker.address.rsm2</name>
-       <value>192.168.40.203:9920</value>
-    </property>
-    <property>
-  <name>yarn.resourcemanager.ha.admin.address.rsm2</name>
-  <value>192.168.40.203:23142</value>
-</property>
-    <!--指定zookeeper集群的地址-->
-    <property>
-       <name>yarn.resourcemanager.zk-address</name>
-        <value>192.168.40.151:2181,192.168.40.152:2181,192.168.40.203:2181</value>
-    </property>
-    <!--启用自动恢复，当任务进行一半，rm坏掉，就要启动自动恢复，默认是false-->
-    <property>
-       <name>yarn.resourcemanager.recovery.enabled</name>
-       <value>true</value>
-    </property>
-    <!--启动nm自动恢复，当集群重启，container开启自动恢复，保障任务的可靠性，默认为false，启动该配置需开启下面配置 -->
- <property>
-       <name>yarn.nodemanager.recovery.enabled</name>
-       <value>true</value>
-    </property>
-   <!--开启nm故障恢复后，nm元数据存储路径 -->
-   <property>
-       <name>yarn.nodemanager.recovery.dir</name>
-       <value>/home/tsg/olap/hadoop-2.7.1/yarn</value>
-    </property>
-   <!--启用nm恢复时监控功能，开启后不会去尝试清理container，默认false -->  
- <property>
-      <name>yarn.nodemanager.recovery.supervised</name>
-      <value>true</value>
-   </property>
-  <!--配置nm可用的RPC地址，默认${yarn.nodemanager.hostname}:0，为临时端口。集群重启后，nm与rm连接的端口会变化，这里指定端口，保障nm restart功能 -->
-<property>
-      <name>yarn.nodemanager.address</name>
-      <value>${yarn.nodemanager.hostname}:9923</value>
-   </property>
-        <property>
-       <name>yarn.resourcemanager.store.class</name>
-       <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
-    </property>
-    <property>
-        <name>yarn.nodemanager.resource.memory-mb</name>
-        <value>30720</value>
-    </property> 
-    <property>
-        <name>yarn.scheduler.minimum-allocation-mb</name>
-        <value>1024</value>
-    </property>
-
-    <property>
-        <name>yarn.scheduler.maximum-allocation-mb</name>
-        <value>30720</value>
-    </property>
-    <!--开启日志聚合 -->
-    <property>
-        <name>yarn.log-aggregation-enable</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>yarn.nodemanager.heartbeat-interval-ms</name>
-        <value>3000</value>
-    </property>
-   <!--日志保留7天 -->
-    <property>
-        <name>yarn.log-aggregation.retain-seconds</name>
-        <value>604800</value>
-    </property>
-    <property>
-        <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
-        <value>3600</value>
-    </property>
-    <property>
-        <name>yarn.nodemanager.remote-app-log-dir</name>
-        <value>/tmp/logs</value>
-    </property>
-   <property>
-        <name>yarn.nodemanager.resource.cpu-vcores</name>
-        <value>14</value>
-    </property>
-    <property>
-        <name>yarn.scheduler.minimum-allocation-vcores</name>
-        <value>1</value>
-    </property>
-    <property>
-        <name>yarn.scheduler.maximum-allocation-vcores</name>
-        <value>14</value>
-    </property>
-    <property>
-        <name>yarn.nodemanager.vmem-check-enabled</name>
-        <value>false</value>
-    </property>
-   <property>
-        <name>yarn.nodemanager.pmem-check-enabled</name>
-        <value>false</value>
-    </property>
-    <property>
-        <name>yarn.nodemanager.disk-health-checker.enable</name>
-        <value>false</value>
-    </property>
-    <!--ApplicationMaster重启次数，配置HA后默认为2-->
-    <property>
-        <name>yarn.resourcemanager.am.max-attempts</name>
-        <value>10000</value>
-    </property>
-    <property>
-        <name>yarn.log.server.url</name>
-        <value>http://bigdata-151:19888/jobhistory/logs</value>
-    </property>
-</configuration>

src/test/java/com/zdjizhi/common/HbaseTest.java

@@ -19,12 +19,12 @@ public class HbaseTest {
     public static void main(String[] args) throws IOException {
         org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create();
 
-        config.set("hbase.zookeeper.quorum", CommonConfig.HBASE_ZOOKEEPER_QUORUM);
+        config.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_QUORUM);
         config.set("hbase.client.retries.number", "3");
         config.set("hbase.bulkload.retries.number", "3");
         config.set("zookeeper.recovery.retry", "3");
-        config.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, CommonConfig.HBASE_CLIENT_OPERATION_TIMEOUT);
+        config.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, FlowWriteConfig.HBASE_CLIENT_OPERATION_TIMEOUT);
-        config.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, CommonConfig.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
+        config.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, FlowWriteConfig.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
 
         TableName tableName  = TableName.valueOf("dos_test");
         Connection conn = ConnectionFactory.createConnection(config);

src/test/java/com/zdjizhi/etl/DosDetectionTest.java

@@ -1,12 +1,12 @@
 package com.zdjizhi.etl;
 
+import com.geedgenetworks.utils.StringUtil;
 import com.zdjizhi.common.DosDetectionThreshold;
 import com.zdjizhi.common.DosEventLog;
 import com.zdjizhi.common.DosSketchLog;
-import com.zdjizhi.utils.IpUtils;
+
-import com.zdjizhi.utils.NacosUtils;
+import com.zdjizhi.utils.IpLookupUtils;
 import com.zdjizhi.utils.SnowflakeId;
-import com.zdjizhi.utils.StringUtil;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;
 
@@ -34,14 +34,14 @@ public class DosDetectionTest {
         serverIpList.add("192.168.50.1/24");
         serverIpList.add("FC::12:0:0/54");
         serverIpList.add("FC::12:0:0");
-        dosDetectionThreshold.setProfileId(4437);
+        dosDetectionThreshold.setProfile_id(4437);
-        dosDetectionThreshold.setAttackType("DNS Flood");
+        dosDetectionThreshold.setAttack_type("DNS Flood");
-        dosDetectionThreshold.setServerIpList(serverIpList);
+        dosDetectionThreshold.setServer_ip_list(serverIpList);
-        dosDetectionThreshold.setSessionsPerSec(1);
+        dosDetectionThreshold.setSessions_per_sec(1);
-        dosDetectionThreshold.setPacketsPerSec(1);
+        dosDetectionThreshold.setPackets_per_sec(1);
-        dosDetectionThreshold.setBitsPerSec(100000);
+        dosDetectionThreshold.setBits_per_sec(100000);
-        dosDetectionThreshold.setIsValid(1);
+        dosDetectionThreshold.setIs_valid(1);
-        dosDetectionThreshold.setSuperiorIds(new Integer[]{5,4,12,27});
+        dosDetectionThreshold.setSuperior_ids(new Integer[]{5,4,12,27});
 
 
         DosSketchLog dosSketchLog = new DosSketchLog ();
@@ -54,9 +54,9 @@ public class DosDetectionTest {
         dosSketchLog.setSource_ip("45.170.244.25");
         dosSketchLog.setDestination_ip("24.152.57.56");
         //静态阈值获取
-        long sessionBase = dosDetectionThreshold.getSessionsPerSec();
+        long sessionBase = dosDetectionThreshold.getSessions_per_sec();
-        long pktBase=dosDetectionThreshold.getPacketsPerSec();
+        long pktBase=dosDetectionThreshold.getPackets_per_sec();
-        long bitBase=dosDetectionThreshold.getBitsPerSec();
+        long bitBase=dosDetectionThreshold.getBits_per_sec();
         //基于速率进行计算
         long diffSession = dosSketchLog.getSketch_sessions() - sessionBase;
         long diffPkt = dosSketchLog.getSketch_packets() - pktBase;
@@ -69,15 +69,15 @@ public class DosDetectionTest {
         long profileId = 0;
         DosEventLog result =null;
         if (diffSessionPercent >= diffPktPercent && diffSessionPercent >= diffBitPercent){
-            profileId = dosDetectionThreshold.getProfileId();
+            profileId = dosDetectionThreshold.getProfile_id();
             result= getDosEventLog(dosSketchLog, sessionBase, diffSession, profileId, STATIC_CONDITION_TYPE, SESSIONS_TAG);
             System.out.println(result);
         }else if (diffPktPercent >= diffSessionPercent && diffPktPercent >= diffBitPercent){
-            profileId = dosDetectionThreshold.getProfileId();
+            profileId = dosDetectionThreshold.getProfile_id();
             result = getDosEventLog(dosSketchLog, pktBase, diffPkt,profileId, STATIC_CONDITION_TYPE, PACKETS_TAG);
             System.out.println(result);
         }else if (diffBitPercent >= diffPktPercent && diffBitPercent >= diffSessionPercent){
-            profileId = dosDetectionThreshold.getProfileId();
+            profileId = dosDetectionThreshold.getProfile_id();
             result = getDosEventLog(dosSketchLog, bitBase, diffByte, profileId, STATIC_CONDITION_TYPE, BITS_TAG);
             System.out.println(result);
         }
@@ -174,7 +174,7 @@ public class DosDetectionTest {
                 String[] ipArr = sourceIpList.split(",");
                 HashSet<String> countrySet = new HashSet<>();
                 for (String ip : ipArr) {
-                    String country = IpUtils.ipLookup.countryLookup(ip);
+                    String country = IpLookupUtils.getCountryLookup(ip);
                     if (StringUtil.isNotBlank(country)){
                         countrySet.add(country);
                     }