修复提交yarn集群依赖冲突bug

pom.xml

@@ -96,8 +96,27 @@
                     </execution>
                 </executions>
             </plugin>
-
         </plugins>
+
+        <resources>
+            <resource>
+                <directory>properties</directory>
+                <includes>
+                    <include>**/*.properties</include>
+                    <include>**/*.xml</include>
+                </includes>
+                <filtering>false</filtering>
+            </resource>
+
+            <resource>
+                <directory>src\main</directory>
+                <includes>
+                    <include>log4j.properties</include>
+                </includes>
+                <filtering>false</filtering>
+            </resource>
+        </resources>
+
     </build>
 
     <dependencies>
@@ -109,17 +128,17 @@
             <version>1.9.3</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.21</version>
-        </dependency>
+        <!--<dependency>-->
+            <!--<groupId>org.slf4j</groupId>-->
+            <!--<artifactId>slf4j-api</artifactId>-->
+            <!--<version>1.7.21</version>-->
+        <!--</dependency>-->
 
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.21</version>
-        </dependency>
+        <!--<dependency>-->
+            <!--<groupId>org.slf4j</groupId>-->
+            <!--<artifactId>slf4j-log4j12</artifactId>-->
+            <!--<version>1.7.21</version>-->
+        <!--</dependency>-->
 
         <dependency>
             <groupId>org.apache.flink</groupId>
@@ -139,6 +158,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>2.7.1</version>
+            <scope>provided</scope>
             <exclusions>
                 <exclusion>
                     <artifactId>zookeeper</artifactId>
@@ -152,6 +172,10 @@
                     <artifactId>guava</artifactId>
                     <groupId>com.google.guava</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -170,6 +194,10 @@
                     <artifactId>log4j-over-slf4j</artifactId>
                     <groupId>org.slf4j</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -244,6 +272,10 @@
                     <groupId>commons-codec</groupId>
                     <artifactId>commons-codec</artifactId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->

src/main/java/com/zdjizhi/etl/DosDetection.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.*;
 import com.zdjizhi.utils.*;
 import inet.ipaddr.IPAddress;
@@ -11,8 +13,6 @@ import org.apache.flink.configuration.Configuration;
 import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap;
 import org.apache.flink.streaming.api.functions.ProcessFunction;
 import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.math.BigDecimal;
 import java.text.NumberFormat;
@@ -26,7 +26,8 @@ import java.util.concurrent.TimeUnit;
  */
 public class DosDetection extends ProcessFunction<DosSketchLog, DosEventLog> {
 
-    private static final Logger logger = LoggerFactory.getLogger(DosDetection.class);
+//    private static final Logger logger = LoggerFactory.getLogger(DosDetection.class);
+    private static final Log logger = LogFactory.get();
     private static Map<String, Map<String, DosBaselineThreshold>> baselineMap = new HashMap<>();
     private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance();
     private HashMap<Integer,HashMap<String, TreeRangeMap<IPAddress, DosDetectionThreshold>>> thresholdRangeMap;

src/main/java/com/zdjizhi/etl/EtlProcessFunction.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosSketchLog;
 import org.apache.commons.lang.StringUtils;
@@ -8,8 +10,6 @@ import org.apache.flink.api.java.tuple.Tuple6;
 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.util.HashSet;
 
@@ -20,7 +20,8 @@ import static com.zdjizhi.sink.OutputStreamSink.outputTag;
  */
 public class EtlProcessFunction extends ProcessWindowFunction<DosSketchLog, DosSketchLog, Tuple3<String,String,Integer>, TimeWindow> {
 
-    private static final Logger logger = LoggerFactory.getLogger(EtlProcessFunction.class);
+//    private static final Logger logger = LoggerFactory.getLogger(EtlProcessFunction.class);
+    private static final Log logger = LogFactory.get();
     private static final String EMPTY_SOURCE_IP_IPV4 = "0.0.0.0";
     private static final String EMPTY_SOURCE_IP_IPV6 = "::";
 

src/main/java/com/zdjizhi/etl/ParseBaselineThreshold.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosBaselineThreshold;
 import com.zdjizhi.utils.DateUtils;
@@ -9,15 +11,14 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.*;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.*;
 
 public class ParseBaselineThreshold {
 
-    private static final Logger logger = LoggerFactory.getLogger(ParseBaselineThreshold.class);
+//    private static final Logger logger = LoggerFactory.getLogger(ParseBaselineThreshold.class);
+    private static final Log logger = LogFactory.get();
     private static ArrayList<String> floodTypeList = new ArrayList<>();
 
     private static Table table = null;

src/main/java/com/zdjizhi/etl/ParseSketchLog.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.fasterxml.jackson.databind.JavaType;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosSketchLog;
@@ -11,8 +13,6 @@ import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
 import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.time.Duration;
 import java.util.ArrayList;
@@ -23,7 +23,8 @@ import java.util.HashMap;
  */
 public class ParseSketchLog {
 
-    private static Logger logger = LoggerFactory.getLogger(ParseSketchLog.class);
+//    private static Logger logger = LoggerFactory.getLogger(ParseSketchLog.class);
+    private static final Log logger = LogFactory.get();
     private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();
     private static JavaType hashmapJsonType = jsonMapperInstance.createCollectionType(HashMap.class, String.class, Object.class);
     private static JavaType listType = jsonMapperInstance.createCollectionType(ArrayList.class, HashMap.class);

src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.fasterxml.jackson.databind.JavaType;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosDetectionThreshold;
@@ -13,8 +15,6 @@ import org.apache.flink.shaded.guava18.com.google.common.collect.Range;
 import org.apache.flink.shaded.guava18.com.google.common.collect.TreeRangeMap;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.message.BasicHeader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.net.URISyntaxException;
 import java.util.ArrayList;
@@ -25,7 +25,8 @@ import java.util.Map;
  * @author wlh
  */
 public class ParseStaticThreshold {
-    private static Logger logger = LoggerFactory.getLogger(ParseStaticThreshold.class);
+//    private static Logger logger = LoggerFactory.getLogger(ParseStaticThreshold.class);
+    private static final Log logger = LogFactory.get();
     private static String encryptpwd;
 
     private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();

src/main/java/com/zdjizhi/etl/TrafficServerIpMetrics.java

@@ -1,15 +1,15 @@
 package com.zdjizhi.etl;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosMetricsLog;
 import com.zdjizhi.common.DosSketchLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 
 class TrafficServerIpMetrics {
 
-    private static final Logger logger = LoggerFactory.getLogger(TrafficServerIpMetrics.class);
+//    private static final Logger logger = LoggerFactory.getLogger(TrafficServerIpMetrics.class);
+    private static final Log logger = LogFactory.get();
 
     static DosMetricsLog getOutputMetric(DosSketchLog midResuleLog) {
         DosMetricsLog dosMetricsLog = new DosMetricsLog();

src/main/java/com/zdjizhi/sink/OutputStreamSink.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.sink;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import com.zdjizhi.common.DosEventLog;
 import com.zdjizhi.common.DosMetricsLog;
@@ -14,14 +16,13 @@ import org.apache.flink.streaming.api.datastream.*;
 import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
 import org.apache.flink.streaming.api.windowing.time.Time;
 import org.apache.flink.util.OutputTag;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * @author 94976
  */
 public class OutputStreamSink {
-    private static final Logger logger = LoggerFactory.getLogger(OutputStreamSink.class);
+//    private static final Logger logger = LoggerFactory.getLogger(OutputStreamSink.class);
+    private static final Log logger = LogFactory.get();
 
     public static OutputTag<DosMetricsLog> outputTag = new OutputTag<DosMetricsLog>("traffic server ip metrics"){};
 

src/main/java/com/zdjizhi/utils/DistributedLock.java

@@ -1,9 +1,9 @@
 package com.zdjizhi.utils;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import org.apache.zookeeper.*;
 import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -17,7 +17,8 @@ import java.util.concurrent.locks.Lock;
 
 
 public class DistributedLock implements Lock, Watcher {
-    private static final Logger logger = LoggerFactory.getLogger(DistributedLock.class);
+//    private static final Logger logger = LoggerFactory.getLogger(DistributedLock.class);
+    private static final Log logger = LogFactory.get();
 
     private ZooKeeper zk = null;
     /**

src/main/java/com/zdjizhi/utils/HttpClientUtils.java

@@ -1,5 +1,7 @@
 package com.zdjizhi.utils;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
 import org.apache.http.*;
 import org.apache.http.client.ClientProtocolException;
@@ -18,8 +20,6 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.message.BasicHeaderElementIterator;
 import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLHandshakeException;
@@ -38,7 +38,8 @@ public class HttpClientUtils {
     /** 全局连接池对象 */
     private static final PoolingHttpClientConnectionManager CONN_MANAGER = new PoolingHttpClientConnectionManager();
 
-    private static Logger logger = LoggerFactory.getLogger(HttpClientUtils.class);
+//    private static Logger logger = LoggerFactory.getLogger(HttpClientUtils.class);
+    private static final Log logger = LogFactory.get();
     public static final String ERROR_MESSAGE = "-1";
 
     /*

src/main/java/com/zdjizhi/utils/NacosUtils.java

@@ -1,11 +1,11 @@
 package com.zdjizhi.utils;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.alibaba.nacos.api.NacosFactory;
 import com.alibaba.nacos.api.PropertyKeyConst;
 import com.alibaba.nacos.api.config.ConfigService;
 import com.alibaba.nacos.api.config.listener.Listener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -13,7 +13,8 @@ import java.util.Properties;
 import java.util.concurrent.Executor;
 
 public class NacosUtils {
-    private static final Logger logger = LoggerFactory.getLogger(NacosUtils.class);
+//    private static final Logger logger = LoggerFactory.getLogger(NacosUtils.class);
+    private static final Log logger = LogFactory.get();
     private static Properties nacosProperties = new Properties();
     private static Properties commonProperties = new Properties();
 

src/main/java/com/zdjizhi/utils/SnowflakeId.java

@@ -1,11 +1,12 @@
 package com.zdjizhi.utils;
 
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import com.zdjizhi.common.CommonConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public class SnowflakeId {
-    private static final Logger logger = LoggerFactory.getLogger(SnowflakeId.class);
+//    private static final Logger logger = LoggerFactory.getLogger(SnowflakeId.class);
+    private static final Log logger = LogFactory.get();
 
     /**
      * 共64位 第一位为符号位 默认0

src/main/java/com/zdjizhi/utils/ZookeeperUtils.java

@@ -1,11 +1,11 @@
 package com.zdjizhi.utils;
 
 import cn.hutool.core.util.StrUtil;
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
 import org.apache.zookeeper.*;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
@@ -13,7 +13,8 @@ import java.util.concurrent.CountDownLatch;
 
 
 public class ZookeeperUtils implements Watcher {
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperUtils.class);
+//    private static final Logger logger = LoggerFactory.getLogger(ZookeeperUtils.class);
+    private static final Log logger = LogFactory.get();
 
     private ZooKeeper zookeeper;
 

src/main/resources/log4j.properties

@@ -0,0 +1,23 @@
+#Log4j
+log4j.rootLogger=info,console,file
+# 控制台日志设置
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=info
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] [Thread\:%t] %l %x - <%m>%n
+
+# 文件日志设置
+log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.file.Threshold=info
+log4j.appender.file.encoding=UTF-8
+log4j.appender.file.Append=true
+#路径请用相对路径，做好相关测试输出到应用目下
+log4j.appender.file.file=${nis.root}/log/flink-dos-detection.log
+log4j.appender.file.DatePattern='.'yyyy-MM-dd
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+#log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss} %X{ip} [%t] %5p %c{1} %m%n
+log4j.appender.file.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] %X{ip} [Thread\:%t]  %l %x - %m%n
+#MyBatis 配置，com.nis.web.dao是mybatis接口所在包
+log4j.logger.com.nis.web.dao=debug
+#bonecp数据源配置
+log4j.category.com.jolbox=debug,console