修改日志输出格式

IP-learning-graph/pom.xml

@@ -42,7 +42,8 @@
         <dependency>
             <groupId>com.arangodb</groupId>
             <artifactId>arangodb-java-driver</artifactId>
-            <version>5.0.4</version>
+            <!--<version>5.0.4</version>-->
+            <version>6.6.3</version>
         </dependency>
     </dependencies>
 

IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseArangoData.java

@@ -1,10 +1,10 @@
 package cn.ac.iie.dao;
 
 import cn.ac.iie.config.ApplicationConfig;
-import cn.ac.iie.etl.ArangoEFqdnAddressIpToMap;
+import cn.ac.iie.etl.fqdn2ip.ArangoEFqdnAddressIpToMap;
-import cn.ac.iie.etl.ArangoEIpVisitFqdnToMap;
+import cn.ac.iie.etl.ip2fqdn.ArangoEIpVisitFqdnToMap;
-import cn.ac.iie.etl.ArangoVFqdnToMap;
+import cn.ac.iie.etl.fqdn.ArangoVFqdnToMap;
-import cn.ac.iie.etl.ArangoVIpToMap;
+import cn.ac.iie.etl.ip.ArangoVIpToMap;
 import cn.ac.iie.utils.ArangoDBConnect;
 import cn.ac.iie.utils.ExecutorThreadPool;
 import com.arangodb.ArangoCursor;
@@ -68,7 +68,7 @@ public class BaseArangoData {
         long maxTime = 0L;
         long diffTime = 0L;
         long startTime = System.currentTimeMillis();
-        LOG.info(sql);
+//        LOG.info(sql);
         ArangoCursor<BaseDocument> timeDoc = arangoDBConnect.executorQuery(sql, BaseDocument.class);
         try {
             if (timeDoc != null){
@@ -78,7 +78,7 @@ public class BaseArangoData {
                     minTime = Long.parseLong(doc.getAttribute("min_time").toString());
                 }
                 long lastTime = System.currentTimeMillis();
-                LOG.info("查询最大最小时间用时：" + (lastTime - startTime));
+                LOG.info(sql+"\n查询最大最小时间用时：" + (lastTime - startTime));
                 diffTime = (maxTime - minTime) / ApplicationConfig.THREAD_POOL_NUMBER;
             }else {
                 LOG.warn("获取ArangoDb时间范围为空");

IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn/ArangoVFqdnToMap.java

@@ -1,4 +1,4 @@
-package cn.ac.iie.etl;
+package cn.ac.iie.etl.fqdn;
 
 import cn.ac.iie.dao.BaseArangoData;
 import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoVFqdnToMap implements Runnable {
     private long diffTime;
     private int threadNumber;
 
-    private ArangoVFqdnToMap(){}
-
     public ArangoVFqdnToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
         this.arangoDBConnect = arangoDBConnect;
         this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoVFqdnToMap implements Runnable {
         long maxThreadTime = finalMinTime + (threadNumber + 1)* diffTime;
         long minThreadTime = finalMinTime + threadNumber * diffTime;
         String query = "FOR doc IN FQDN filter doc.FIRST_FOUND_TIME >= "+minThreadTime+" and doc.FIRST_FOUND_TIME <= "+maxThreadTime+" RETURN doc";
-        LOG.info(name+":"+query);
+//        LOG.info(name+":"+query);
         long s = System.currentTimeMillis();
         ArangoCursor<BaseDocument> docs = arangoDBConnect.executorQuery(query, BaseDocument.class);
 
@@ -44,9 +42,9 @@ public class ArangoVFqdnToMap implements Runnable {
                     BaseArangoData.v_Fqdn_Map.put(key,doc);
                     i++;
                 }
-                LOG.info(name+":共处理数据"+ i);
+//                LOG.info(name+":共处理FQDN数据"+ i);
                 long l = System.currentTimeMillis();
-                LOG.info(name+"运行时间："+(l-s));
+                LOG.info(query+"\n处理FQDN数据"+ i+"条,运行时间："+(l-s));
             }else {
                 LOG.warn("获取VFqdn异常，结果为空");
             }

IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn/UpdateVFqdn.java

@@ -1,4 +1,4 @@
-package cn.ac.iie.etl;
+package cn.ac.iie.etl.fqdn;
 
 import cn.ac.iie.config.ApplicationConfig;
 import cn.ac.iie.dao.BaseArangoData;
@@ -8,8 +8,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Set;
 
 public class UpdateVFqdn implements Runnable{
     private static final Logger LOG = LoggerFactory.getLogger(UpdateVFqdn.class);

IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn2ip/ArangoEFqdnAddressIpToMap.java

@@ -1,4 +1,4 @@
-package cn.ac.iie.etl;
+package cn.ac.iie.etl.fqdn2ip;
 
 import cn.ac.iie.dao.BaseArangoData;
 import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
     private long diffTime;
     private int threadNumber;
 
-    private ArangoEFqdnAddressIpToMap(){}
-
     public ArangoEFqdnAddressIpToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
         this.arangoDBConnect = arangoDBConnect;
         this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
         long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
         long minThreadTime = finalMinTime + threadNumber * diffTime;
         String query = "FOR doc IN R_LOCATE_FQDN2IP filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
-        LOG.info(name + ":" + query);
+//        LOG.info(name + ":" + query);
         long s = System.currentTimeMillis();
         try {
             ArangoCursor<BaseEdgeDocument> docs = arangoDBConnect.executorQuery(query, BaseEdgeDocument.class);
@@ -43,9 +41,8 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
                     BaseArangoData.e_Fqdn_Address_Ip_Map.put(key, doc);
                     i++;
                 }
-                LOG.info(name + ":共处理数据" + i);
                 long l = System.currentTimeMillis();
-                LOG.info(name + "运行时间：" + (l - s));
+                LOG.info(query+ "\n处理R_LOCATE_FQDN2IP数据" + i + "条，运行时间：" + (l - s));
             }else {
                 LOG.warn("查询R_LOCATE_FQDN2IP异常,结果为空");
             }

IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn2ip/UpdateEFqdnAddressIp.java

@@ -1,4 +1,4 @@
-package cn.ac.iie.etl;
+package cn.ac.iie.etl.fqdn2ip;
 
 import cn.ac.iie.config.ApplicationConfig;
 import cn.ac.iie.dao.BaseArangoData;

IP-learning-graph/src/main/java/cn/ac/iie/etl/ip/ArangoVIpToMap.java

@@ -1,4 +1,4 @@
-package cn.ac.iie.etl;
+package cn.ac.iie.etl.ip;
 
 import cn.ac.iie.dao.BaseArangoData;
 import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoVIpToMap implements Runnable {
     private long diffTime;
     private int threadNumber;
 
-    private ArangoVIpToMap() {}
-
     public ArangoVIpToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
         this.arangoDBConnect = arangoDBConnect;
         this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoVIpToMap implements Runnable {
         long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
         long minThreadTime = finalMinTime + threadNumber * diffTime;
         String query = "FOR doc IN IP filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
-        LOG.info(name + ":" + query);
+//        LOG.info(name + ":" + query);
         long s = System.currentTimeMillis();
         try {
             ArangoCursor<BaseDocument> docs = arangoDBConnect.executorQuery(query, BaseDocument.class);
@@ -43,9 +41,8 @@ public class ArangoVIpToMap implements Runnable {
                     BaseArangoData.v_Ip_Map.put(key, doc);
                     i++;
                 }
-                LOG.info(name + ":共处理数据" + i);
                 long l = System.currentTimeMillis();
-                LOG.info(name + "运行时间：" + (l - s));
+                LOG.info(query+ "\n处理IP数据" + i + "条，运行时间：" + (l - s));
             }else {
                 LOG.warn("获取VIP异常，结果为空");
             }

IP-learning-graph/src/main/java/cn/ac/iie/etl/ip/UpdateVIP.java

@@ -1,4 +1,4 @@
-package cn.ac.iie.etl;
+package cn.ac.iie.etl.ip;
 
 
 import cn.ac.iie.config.ApplicationConfig;

IP-learning-graph/src/main/java/cn/ac/iie/etl/ip2fqdn/ArangoEIpVisitFqdnToMap.java

@@ -1,4 +1,4 @@
-package cn.ac.iie.etl;
+package cn.ac.iie.etl.ip2fqdn;
 
 import cn.ac.iie.dao.BaseArangoData;
 import cn.ac.iie.utils.ArangoDBConnect;
@@ -16,8 +16,6 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
     private long diffTime;
     private int threadNumber;
 
-    private ArangoEIpVisitFqdnToMap(){}
-
     public ArangoEIpVisitFqdnToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
         this.arangoDBConnect = arangoDBConnect;
         this.finalMinTime = finalMinTime;
@@ -30,7 +28,7 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
         long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
         long minThreadTime = finalMinTime + threadNumber * diffTime;
         String query = "FOR doc IN R_VISIT_IP2FQDN filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
-        LOG.info(name + ":" + query);
+//        LOG.info(name + ":" + query);
         long s = System.currentTimeMillis();
         ArangoCursor<BaseEdgeDocument> docs = arangoDBConnect.executorQuery(query, BaseEdgeDocument.class);
 
@@ -42,9 +40,8 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
                 BaseArangoData.e_Ip_Visit_Fqdn_Map.put(key, doc);
                 i++;
             }
-            LOG.info(name + ":共处理数据" + i);
             long l = System.currentTimeMillis();
-            LOG.info(name + "运行时间：" + (l - s));
+            LOG.info(query+ "\n处理R_VISIT_IP2FQDN数据" + i + "条，运行时间：" + (l - s));
         }else {
             LOG.warn("查询R_VISIT_IP2FQDN异常，结果为空");
         }

IP-learning-graph/src/main/java/cn/ac/iie/etl/ip2fqdn/UpdateEIpVisitFqdn.java

@@ -1,4 +1,4 @@
-package cn.ac.iie.etl;
+package cn.ac.iie.etl.ip2fqdn;
 
 import cn.ac.iie.config.ApplicationConfig;
 import cn.ac.iie.dao.BaseArangoData;
@@ -8,7 +8,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Set;
 

IP-learning-graph/src/main/java/cn/ac/iie/pojo/VertexFqdn.java

@@ -0,0 +1,5 @@
+package cn.ac.iie.pojo;
+
+public class VertexFqdn {
+
+}

IP-learning-graph/src/main/java/cn/ac/iie/test/IpLearningApplicationTest.java

@@ -58,7 +58,6 @@ public class IpLearningApplicationTest {
                 LOG.error("主线程阻塞异常：\n"+e.toString());
             }
 
-            //        BaseClickhouseData.BaseEFqdnAddressIp();
             long lastC = System.currentTimeMillis();
             LOG.info("更新ArangoDb时间："+(lastC - startC));
         }catch (Exception e){

IP-learning-graph/src/main/java/cn/ac/iie/utils/ArangoDBConnect.java

@@ -42,7 +42,7 @@ public class ArangoDBConnect {
         return conn;
     }
 
-    public ArangoDatabase getDatabase(){
+    private ArangoDatabase getDatabase(){
         return arangoDB.db(ApplicationConfig.ARANGODB_DB_NAME);
     }
 
@@ -59,7 +59,8 @@ public class ArangoDBConnect {
     public <T> ArangoCursor<T> executorQuery(String query,Class<T> type){
         ArangoDatabase database = getDatabase();
         Map<String, Object> bindVars = new MapBuilder().get();
-        AqlQueryOptions options = new AqlQueryOptions().ttl(ApplicationConfig.ARANGODB_TTL);
+        AqlQueryOptions options = new AqlQueryOptions()
+                .ttl(ApplicationConfig.ARANGODB_TTL);
         try {
             return database.query(query, bindVars, options, type);
         }catch (Exception e){
@@ -70,6 +71,7 @@ public class ArangoDBConnect {
         }
     }
 
+    @Deprecated
     public <T> void insertAndUpdate(ArrayList<T> docInsert,ArrayList<T> docUpdate,String collectionName){
         ArangoDatabase database = getDatabase();
         try {
@@ -97,17 +99,16 @@ public class ArangoDBConnect {
             if (!docOverwrite.isEmpty()){
                 DocumentCreateOptions documentCreateOptions = new DocumentCreateOptions();
                 documentCreateOptions.overwrite(true);
-                documentCreateOptions.returnNew(true);
+                documentCreateOptions.silent(true);
-                documentCreateOptions.returnOld(true);
                 MultiDocumentEntity<DocumentCreateEntity<T>> documentCreateEntityMultiDocumentEntity = collection.insertDocuments(docOverwrite, documentCreateOptions);
                 Collection<ErrorEntity> errors = documentCreateEntityMultiDocumentEntity.getErrors();
                 for (ErrorEntity errorEntity:errors){
                     LOG.error("写入arangoDB异常："+errorEntity.getErrorMessage());
                 }
-
             }
         }catch (Exception e){
-            LOG.error(e.toString());
+            LOG.error("更新失败："+e.toString());
+//            clean();
         }finally {
             docOverwrite.clear();
         }

IP-learning-graph/src/main/resources/application.properties

@@ -1,5 +1,5 @@
 #arangoDB参数配置
-arangoDB.host=192.168.40.182
+arangoDB.host=192.168.40.127
 arangoDB.port=8529
 arangoDB.user=root
 arangoDB.password=111111

IP-learning-graph/src/test/java/cn/ac/iie/IpTest.java

@@ -1,10 +1,15 @@
 package cn.ac.iie;
 
 import cn.ac.iie.config.ApplicationConfig;
+import cn.ac.iie.dao.BaseClickhouseData;
+import com.arangodb.entity.BaseDocument;
 
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 public class IpTest {
@@ -17,7 +22,7 @@ public class IpTest {
             System.out.println(s);
             System.out.println(pattern.matcher(s).matches());
         }
-*/
+
 //        String ip = "17.57.145.7";
         String ip = "pixel.rubiconproject.com";
 //        String ip = "113.200.17.239";
@@ -25,6 +30,23 @@ public class IpTest {
         int hash = Math.abs(ip.hashCode());
         int i = hash % ApplicationConfig.THREAD_POOL_NUMBER;
         System.out.println(i);
+        */
+        HashMap<Integer, ArrayList<BaseDocument>> vFqdnMap = new HashMap<>();
+        for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++) {
+            vFqdnMap.put(i, new ArrayList<>());
+        }
+        try {
+//        ArrayList<BaseDocument> orDefault = vFqdnMap.getOrDefault(-7, new ArrayList<>());
+            ArrayList<BaseDocument> orDefault = vFqdnMap.get(-7);
+            System.out.println(orDefault.toString());
+            Set<Integer> integers = vFqdnMap.keySet();
+            for (Integer in:integers){
+                System.out.println(in);
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+            System.out.println(e.toString());
+        }
 //        String[] ipArr = ipStr.split("\\.");
 //        long ipLong = (Long.valueOf(ipArr[0]) << 24) + (Long.valueOf(ipArr[1]) << 16) + (Long.valueOf(ipArr[2]) << 8) + (Long.valueOf(ipArr[3]));
 //        System.out.println(ipLong);

IP-learning-graph/src/test/java/cn/ac/iie/TestMap.java

@@ -1,17 +1,12 @@
 package cn.ac.iie;
 
-import cn.ac.iie.config.ApplicationConfig;
+import cn.ac.iie.dao.BaseArangoData;
-import cn.ac.iie.etl.UpdateEFqdnAddressIp;
 import cn.ac.iie.utils.ArangoDBConnect;
-import cn.ac.iie.utils.ClickhouseConnect;
+import cn.ac.iie.utils.ExecutorThreadPool;
-import com.alibaba.druid.pool.DruidPooledConnection;
-import com.arangodb.ArangoCollection;
-import com.arangodb.ArangoDatabase;
 import com.arangodb.entity.*;
 
-import java.sql.ResultSet;
-import java.sql.Statement;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 
 public class TestMap {
 
@@ -52,6 +47,7 @@ public class TestMap {
         */
 
 
+
         String[] distCipRecents = new  String[]{"2.3"};
         ArrayList<BaseDocument> baseEdgeDocuments = new ArrayList<>();
 
@@ -77,5 +73,38 @@ public class TestMap {
 
 
 
+        /*
+
+        BaseArangoData.BaseEFqdnAddressIpDataMap();
+
+        ExecutorThreadPool.shutdown();
+        ExecutorThreadPool.awaitThreadTask();
+
+        try {
+            ConcurrentHashMap.KeySetView<String, BaseEdgeDocument> keySet = BaseArangoData.e_Fqdn_Address_Ip_Map.keySet();
+            ArrayList<BaseEdgeDocument> baseEdgeDocuments = new ArrayList<>();
+            ArangoDBConnect instance = ArangoDBConnect.getInstance();
+            for (String key:keySet){
+                BaseEdgeDocument baseEdgeDocument = BaseArangoData.e_Fqdn_Address_Ip_Map.get(key);
+//                Long[] tls_cnt_recents = (Long[]) baseEdgeDocument.getAttribute("TLS_CNT_RECENT");
+                ArrayList<Long> tlsCntRecent = (ArrayList<Long>) baseEdgeDocument.getAttribute("TLS_CNT_RECENT");
+                Long[] tlsCntRecentsSrc = tlsCntRecent.toArray(new Long[tlsCntRecent.size()]);
+                System.out.println(Arrays.toString(tlsCntRecentsSrc));
+                tlsCntRecent.set(tlsCntRecent.size()-2,99L);
+
+                baseEdgeDocument.addAttribute("TLS_CNT_RECENT",tlsCntRecent);
+                baseEdgeDocuments.add(baseEdgeDocument);
+            }
+            instance.overwrite(baseEdgeDocuments,"R_LOCATE_FQDN2IP");
+        }catch (Exception e){
+            e.printStackTrace();
+        }finally {
+            ArangoDBConnect.clean();
+        }
+
+*/
+
+
+
     }
 }