client IP只根据阈值统计，去掉最近24小时限制

IP-learning-graph/pom.xml

@@ -39,12 +39,6 @@
             <version>1.2.1</version>
         </dependency>
 
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-        </dependency>
-
         <dependency>
             <groupId>com.arangodb</groupId>
             <artifactId>arangodb-java-driver</artifactId>

IP-learning-graph/src/main/java/cn/ac/iie/config/ApplicationConfig.java

@@ -14,7 +14,7 @@ public class ApplicationConfig {
     public static final Integer ARANGODB_BATCH = ConfigUtils.getIntProperty( "arangoDB.batch");
 
     public static final Integer UPDATE_ARANGO_BATCH = ConfigUtils.getIntProperty("update.arango.batch");
-    public static final String ARANGODB_READ_LIMIT = ConfigUtils.getStringProperty("arangoDB.read.limit");
+    public static final Long ARANGODB_READ_LIMIT = ConfigUtils.getLongProperty("arangoDB.read.limit");
 
     public static final Integer THREAD_POOL_NUMBER = ConfigUtils.getIntProperty( "thread.pool.number");
     public static final Integer THREAD_AWAIT_TERMINATION_TIME = ConfigUtils.getIntProperty( "thread.await.termination.time");

IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseArangoData.java

@@ -58,57 +58,6 @@ public class BaseArangoData {
         }
     }
 
-    public <T extends BaseDocument> ConcurrentHashMap<Integer,ConcurrentHashMap<String, T>> readHistoryData(String table, Class<T> type){
-        ConcurrentHashMap<Integer,ConcurrentHashMap<String, T>> map = new ConcurrentHashMap<>();
-        try {
-            LOG.info("开始更新"+table);
-            long start = System.currentTimeMillis();
-            for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++){
-                map.put(i,new ConcurrentHashMap<>());
-            }
-            CountDownLatch countDownLatch = new CountDownLatch(ApplicationConfig.THREAD_POOL_NUMBER);
-            Long countTotal = getCountTotal(table);
-            for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++) {
-                String sql = getQuerySql(countTotal, i, table);
-                ReadHistoryArangoData<T> readHistoryArangoData =
-                        new ReadHistoryArangoData<>(arangoDBConnect, sql, map,type,table,countDownLatch);
-                threadPool.executor(readHistoryArangoData);
-            }
-            countDownLatch.await();
-            long last = System.currentTimeMillis();
-            LOG.info("读取"+table+" arangoDB 共耗时："+(last-start));
-        }catch (Exception e){
-            e.printStackTrace();
-            LOG.error("读取历史数据失败 "+e.toString());
-        }
-        return map;
-    }
-
-    private long[] getTimeRange(String table){
-        long minTime = 0L;
-        long maxTime = 0L;
-        long startTime = System.currentTimeMillis();
-        String sql = "LET doc = (FOR doc IN "+table+" RETURN doc) return {max_time:MAX(doc[*].FIRST_FOUND_TIME),min_time:MIN(doc[*].FIRST_FOUND_TIME)}";
-        ArangoCursor<BaseDocument> timeDoc = arangoDBConnect.executorQuery(sql, BaseDocument.class);
-        try {
-            if (timeDoc != null){
-                while (timeDoc.hasNext()) {
-                    BaseDocument doc = timeDoc.next();
-                    maxTime = Long.parseLong(doc.getAttribute("max_time").toString()) + ApplicationConfig.THREAD_POOL_NUMBER;
-                    minTime = Long.parseLong(doc.getAttribute("min_time").toString());
-                }
-                long lastTime = System.currentTimeMillis();
-                LOG.info(sql+"\n查询最大最小时间用时：" + (lastTime - startTime));
-            }else {
-                LOG.warn("获取ArangoDb时间范围为空");
-            }
-        }catch (Exception e){
-            e.printStackTrace();
-        }
-        return new long[]{minTime, maxTime};
-
-    }
-
     private Long getCountTotal(String table){
         long start = System.currentTimeMillis();
         Long cnt = 0L;
@@ -129,17 +78,12 @@ public class BaseArangoData {
     private String getQuerySql(Long cnt,int threadNumber, String table){
         long sepNum = cnt / ApplicationConfig.THREAD_POOL_NUMBER + 1;
         long offsetNum = threadNumber * sepNum;
+        if (sepNum >= ApplicationConfig.ARANGODB_READ_LIMIT * 10000){
+            sepNum = ApplicationConfig.ARANGODB_READ_LIMIT * 10000;
+        }
         return "FOR doc IN " + table + " limit "+offsetNum+","+sepNum+" RETURN doc";
     }
 
 
-    private String getQuerySql(long[] timeRange,int threadNumber,String table){
-        long minTime = timeRange[0];
-        long maxTime = timeRange[1];
-        long diffTime = (maxTime - minTime) / ApplicationConfig.THREAD_POOL_NUMBER;
-        long maxThreadTime = minTime + (threadNumber + 1)* diffTime;
-        long minThreadTime = minTime + threadNumber * diffTime;
-        return "FOR doc IN "+table+" filter doc.FIRST_FOUND_TIME >= "+minThreadTime+" and doc.FIRST_FOUND_TIME <= "+maxThreadTime+" " + ApplicationConfig.ARANGODB_READ_LIMIT + " RETURN doc";
-    }
 
 }

IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseClickhouseData.java

@@ -73,51 +73,5 @@ public class BaseClickhouseData {
         }
     }
 
-    public <T extends BaseDocument> HashMap<Integer, HashMap<String,ArrayList<T>>> baseDocFromCk(Supplier<String> getSqlSupplier,
-                                                             Function<ResultSet,T> formatResultFunc){
-        long start = System.currentTimeMillis();
-        HashMap<Integer, HashMap<String, ArrayList<T>>> newDataMap = initializeMap();
-        if (newDataMap == null){
-            return null;
-        }
-        String sql = getSqlSupplier.get();
-        try {
-            connection = manger.getConnection();
-            statement = connection.createStatement();
-            ResultSet resultSet = statement.executeQuery(sql);
-            int i = 0;
-            while (resultSet.next()) {
-                T newDoc = formatResultFunc.apply(resultSet);
-                if (newDoc != null) {
-                    i+=1;
-                    putMapByHashcode(newDoc, newDataMap);
-                }
-            }
-            long last = System.currentTimeMillis();
-            LOG.info(sql + "\n读取"+i+"条数据，运行时间：" + (last - start));
-        }catch (Exception e){
-            e.printStackTrace();
-            LOG.error("获取原始数据失败 "+e.toString());
-        }finally {
-            manger.clear(statement,connection);
-        }
-        return newDataMap;
-    }
-
-    private <T extends BaseDocument> HashMap<Integer, HashMap<String,ArrayList<T>>> initializeMap(){
-        try {
-            HashMap<Integer, HashMap<String, ArrayList<T>>> newDataMap = new HashMap<>();
-            for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++) {
-                newDataMap.put(i, new HashMap<>());
-            }
-            return newDataMap;
-        }catch (Exception e){
-            e.printStackTrace();
-            LOG.error("数据初始化失败 "+e.toString());
-            return null;
-        }
-
-    }
-
 
 }

IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java

@@ -44,8 +44,8 @@ public class UpdateGraphData {
         long start = System.currentTimeMillis();
         try {
 
-            updateDocument(newVertexFqdnMap, historyVertexFqdnMap, "FQDN", Fqdn.class,BaseDocument.class,
+//            updateDocument(newVertexFqdnMap, historyVertexFqdnMap, "FQDN", Fqdn.class,BaseDocument.class,
-                    ReadClickhouseData::getVertexFqdnSql,ReadClickhouseData::getVertexFqdnDocument);
+//                    ReadClickhouseData::getVertexFqdnSql,ReadClickhouseData::getVertexFqdnDocument);
 
             updateDocument(newVertexIpMap,historyVertexIpMap,"IP", Ip.class,BaseDocument.class,
                     ReadClickhouseData::getVertexIpSql,ReadClickhouseData::getVertexIpDocument);
@@ -75,41 +75,6 @@ public class UpdateGraphData {
         }
     }
 
-    public void updateArango2(){
-        long start = System.currentTimeMillis();
-        try {
-
-            updateDocument("FQDN", Fqdn.class,BaseDocument.class,
-                    ReadClickhouseData::getVertexFqdnSql,ReadClickhouseData::getVertexFqdnDocument);
-
-            updateDocument("IP", Ip.class,BaseDocument.class,
-                    ReadClickhouseData::getVertexIpSql,ReadClickhouseData::getVertexIpDocument);
-
-            updateDocument("SUBSCRIBER", Subscriber.class,BaseDocument.class,
-                    ReadClickhouseData::getVertexSubscriberSql,ReadClickhouseData::getVertexSubscriberDocument);
-
-            updateDocument("R_LOCATE_FQDN2IP", LocateFqdn2Ip.class,BaseEdgeDocument.class,
-                    ReadClickhouseData::getRelationshipFqdnAddressIpSql,ReadClickhouseData::getRelationFqdnAddressIpDocument);
-
-//            updateDocument("R_VISIT_IP2FQDN",
-//                    VisitIp2Fqdn.class,BaseEdgeDocument.class,
-//                    ReadClickhouseData::getRelationshipIpVisitFqdnSql,ReadClickhouseData::getRelationIpVisitFqdnDocument);
-
-            updateDocument("R_LOCATE_SUBSCRIBER2IP",
-                    LocateSubscriber2Ip.class,BaseEdgeDocument.class,
-                    ReadClickhouseData::getRelationshipSubsciberLocateIpSql,ReadClickhouseData::getRelationshipSubsciberLocateIpDocument);
-
-
-            long last = System.currentTimeMillis();
-            LOG.info("iplearning application运行完毕，用时："+(last - start));
-        }catch (Exception e){
-            e.printStackTrace();
-        }finally {
-            arangoManger.clean();
-            pool.shutdown();
-        }
-    }
-
 
     private <T extends BaseDocument> void updateDocument(HashMap<Integer, HashMap<String, ArrayList<T>>> newMap,
                                                          ConcurrentHashMap<Integer, ConcurrentHashMap<String, T>> historyMap,
@@ -151,45 +116,5 @@ public class UpdateGraphData {
         }
     }
 
-    private <T extends BaseDocument> void updateDocument(String collection,
-                                                         Class<? extends Document<T>> taskType,
-                                                         Class<T> docmentType,
-                                                         Supplier<String> getSqlSupplier,
-                                                         Function<ResultSet,T> formatResultFunc){
-        ConcurrentHashMap<Integer, ConcurrentHashMap<String, T>> historyData = baseArangoData.readHistoryData(collection, docmentType);
-        LOG.info(collection+" 读取clickhouse,封装结果集");
-        HashMap<Integer, HashMap<String, ArrayList<T>>> newData = baseClickhouseData.baseDocFromCk(getSqlSupplier, formatResultFunc);
-        try {
-            LOG.info(collection+" 开始更新");
-            long start = System.currentTimeMillis();
-            CountDownLatch countDownLatch = new CountDownLatch(ApplicationConfig.THREAD_POOL_NUMBER);
-            for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++){
-                HashMap<String, ArrayList<T>> tmpNewMap = newData.get(i);
-                ConcurrentHashMap<String, T> tmpHisMap = historyData.get(i);
-                Constructor constructor = taskType.getConstructor(
-                        HashMap.class,
-                        ArangoDBConnect.class,
-                        String.class,
-                        ConcurrentHashMap.class,
-                        CountDownLatch.class);
-                Document docTask = (Document)constructor.newInstance(tmpNewMap, arangoManger, collection, tmpHisMap, countDownLatch);
-                pool.executor(docTask);
-            }
-            countDownLatch.await();
-            long last = System.currentTimeMillis();
-            LOG.info(collection+" 更新完毕,共耗时："+(last-start));
-
-        }catch (Exception e){
-            e.printStackTrace();
-            LOG.error("更新"+collection+"失败！！"+e.toString());
-        }finally {
-            newData.clear();
-            historyData.clear();
-        }
-
-
-    }
-
-
 
 }

IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java

@@ -66,9 +66,11 @@ public class ReadClickhouseData {
             long bytesSum = resultSet.getLong("BYTES_SUM");
             String ipType = resultSet.getString("ip_type");
             String[] commonLinkInfos = (String[]) resultSet.getArray("common_link_info").getArray();
-            String commonLinkInfo = "";
+            String commonLinkInfo;
-            if (commonLinkInfos.length > 1){
+            if (commonLinkInfos.length > 1 && !commonLinkInfos[1].equals("")){
                 commonLinkInfo = commonLinkInfos[1];
+            }else {
+                commonLinkInfo = commonLinkInfos[0];
             }
             newDoc.setKey(ip);
             newDoc.addAttribute("IP", ip);
@@ -263,9 +265,10 @@ public class ReadClickhouseData {
     }
 
     public static String getVertexIpSql() {
-        String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime;
+//        String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime + " AND common_schema_type != 'BASE'";
-        String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
+        String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime ;
-        String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
+        String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_c2s) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
+        String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_s2c) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
         return "SELECT * FROM((" + clientIpSql + ") UNION ALL (" + serverIpSql + "))";
     }
 

IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java

@@ -8,6 +8,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
@@ -58,7 +59,7 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
                     switch (table) {
                         case "R_LOCATE_FQDN2IP":
                             updateProtocolDocument(doc);
-                            deleteDistinctClientIpByTime(doc);
+//                            deleteDistinctClientIpByTime(doc);
                             list.add(doc);
                             break;
                         default:
@@ -73,7 +74,7 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
                 LOG.info(query + "\n读取" + i + "条数据，运行时间：" + (l - s));
             }
         } catch (Exception e) {
-            e.printStackTrace();
+            LOG.error(Arrays.toString(e.getStackTrace()));
         } finally {
             countDownLatch.countDown();
             LOG.info("本线程读取完毕，剩余线程数量：" + countDownLatch.getCount());
@@ -97,6 +98,11 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
     private void deleteDistinctClientIpByTime(T doc) {
         ArrayList<String> distCip = (ArrayList<String>) doc.getAttribute("DIST_CIP");
         ArrayList<Long> distCipTs = (ArrayList<Long>) doc.getAttribute("DIST_CIP_TS");
+        if (distCip == null || distCip.isEmpty()){
+            doc.updateAttribute("DIST_CIP", new String[0]);
+            doc.updateAttribute("DIST_CIP_TS", new long[0]);
+            return;
+        }
         distCipTs.add(currentHour - RECENT_COUNT_HOUR * 3600);
         Collections.sort(distCipTs);
         Collections.reverse(distCipTs);

IP-learning-graph/src/main/java/cn/ac/iie/service/update/relationship/LocateFqdn2Ip.java

@@ -53,12 +53,12 @@ public class LocateFqdn2Ip extends Relationship {
     }
 
     private void updateDistinctClientIp(BaseEdgeDocument newEdgeDocument,BaseEdgeDocument edgeDocument){
-        String[] distCip = (String[]) edgeDocument.getAttribute("DIST_CIP");
+        ArrayList<String> distCip = (ArrayList<String>) edgeDocument.getAttribute("DIST_CIP");
-        long[] distCipTs = (long[]) edgeDocument.getAttribute("DIST_CIP_TS");
+        ArrayList<Long> distCipTs = (ArrayList<Long>) edgeDocument.getAttribute("DIST_CIP_TS");
         HashMap<String, Long> distCipToTs = new HashMap<>();
-        if (distCip.length == distCipTs.length){
+        if (distCip.size() == distCipTs.size()){
-            for (int i = 0;i < distCip.length;i++){
+            for (int i = 0;i < distCip.size();i++){
-                distCipToTs.put(distCip[i],distCipTs[i]);
+                distCipToTs.put(distCip.get(i),distCipTs.get(i));
             }
         }
         Object[] distCipRecent = (Object[])newEdgeDocument.getAttribute("DIST_CIP");

IP-learning-graph/src/main/java/cn/ac/iie/test/IpLearningApplicationTest.java

@@ -6,8 +6,7 @@ public class IpLearningApplicationTest {
 
     public static void main(String[] args) {
         UpdateGraphData updateGraphData = new UpdateGraphData();
-//        updateGraphData.updateArango();
+        updateGraphData.updateArango();
-        updateGraphData.updateArango2();
 
     }
 }

IP-learning-graph/src/main/resources/application.properties

@@ -1,27 +1,25 @@
 #arangoDB参数配置
-arangoDB.host=192.168.40.182
+arangoDB.host=192.168.44.12
-#arangoDB.host=192.168.40.224
 arangoDB.port=8529
 arangoDB.user=upsert
 arangoDB.password=ceiec2018
-arangoDB.DB.name=ip-learning-test
+arangoDB.DB.name=tsg_galaxy_v3_test
-#arangoDB.DB.name=tsg_galaxy_v3
 arangoDB.batch=100000
 arangoDB.ttl=3600
 
-arangoDB.read.limit=
+arangoDB.read.limit=10
 update.arango.batch=10000
 
-thread.pool.number=10
+thread.pool.number=40
 thread.await.termination.time=10
 
 
 #读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围
-time.limit.type=0
+time.limit.type=1
-read.clickhouse.max.time=1596684142
+read.clickhouse.max.time=1603421554
-read.clickhouse.min.time=1596425769
+read.clickhouse.min.time=1603354682
 
 update.interval=3600
-distinct.client.ip.num=10000
+distinct.client.ip.num=100
 recent.count.hour=24
 

IP-learning-graph/src/main/resources/clickhouse.properties

@@ -1,9 +1,7 @@
 drivers=ru.yandex.clickhouse.ClickHouseDriver
 mdb.user=default
-db.id=192.168.40.186:8123/tsg_galaxy_v3?socket_timeout=300000
+db.id=192.168.44.10:8123/tsg_galaxy_v3?socket_timeout=3600000&compress=0
-mdb.password=111111
+mdb.password=ceiec2019
-#db.id=192.168.40.224:8123/tsg_galaxy_v3?socket_timeout=300000
-#mdb.password=ceiec2019
 initialsize=1
 minidle=1
 maxactive=50

IP-learning-graph/src/main/resources/log4j.properties

@@ -17,7 +17,6 @@ log4j.appender.file.encoding=UTF-8
 log4j.appender.file.Append=true
 #·<><C2B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>·<EFBFBD><C2B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ز<EFBFBD><D8B2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ӧ<EFBFBD><D3A6>Ŀ<EFBFBD><C4BF>
 #log4j.appender.file.file=/home/ceiec/iplearning/logs/ip-learning-application.log
-#log4j.appender.file.file=/home/ceiec/iplearning/testLog/ip-learning-application.log
 log4j.appender.file.file=./logs/ip-learning-application.log
 log4j.appender.file.DatePattern='.'yyyy-MM-dd
 log4j.appender.file.layout=org.apache.log4j.PatternLayout

IP-learning-graph/src/test/java/cn/ac/iie/TestReadArango.java

@@ -1,53 +0,0 @@
-package cn.ac.iie;
-
-import cn.ac.iie.dao.BaseArangoData;
-import cn.ac.iie.utils.ArangoDBConnect;
-import cn.ac.iie.utils.ExecutorThreadPool;
-import com.arangodb.entity.BaseDocument;
-import com.arangodb.entity.BaseEdgeDocument;
-import org.junit.After;
-import org.junit.Test;
-
-import java.util.Enumeration;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class TestReadArango {
-    private static ExecutorThreadPool pool = ExecutorThreadPool.getInstance();
-    private static ArangoDBConnect arangoManger = ArangoDBConnect.getInstance();
-
-    private static BaseArangoData baseArangoData = new BaseArangoData();
-
-
-    @Test
-    public void testReadFqdnFromArango() {
-        ConcurrentHashMap<Integer, ConcurrentHashMap<String, BaseDocument>> historyData =
-                baseArangoData.readHistoryData("FQDN", BaseDocument.class);
-        printMap(historyData);
-    }
-
-    @Test
-    public void testReadFqdnLocIpFromArango() {
-        ConcurrentHashMap<Integer, ConcurrentHashMap<String, BaseEdgeDocument>> ip =
-                baseArangoData.readHistoryData("R_LOCATE_FQDN2IP", BaseEdgeDocument.class);
-        printMap(ip);
-    }
-
-    private <T extends BaseDocument> void printMap(ConcurrentHashMap<Integer, ConcurrentHashMap<String, T>> historyData) {
-        ConcurrentHashMap<String, T> map = historyData.get(2);
-        Enumeration<String> keys = map.keys();
-        while (keys.hasMoreElements()) {
-            String key = keys.nextElement();
-            T document = map.get(key);
-            System.out.println(document.toString());
-        }
-    }
-
-
-    @After
-    public void clearSource() {
-        pool.shutdown();
-        arangoManger.clean();
-    }
-
-
-}

IP-learning-graph/src/test/java/cn/ac/iie/TestReadClickhouse.java

@@ -1,46 +0,0 @@
-package cn.ac.iie;
-
-import cn.ac.iie.dao.BaseClickhouseData;
-import cn.ac.iie.service.ingestion.ReadClickhouseData;
-import com.arangodb.entity.BaseDocument;
-import com.arangodb.entity.BaseEdgeDocument;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Set;
-
-public class TestReadClickhouse {
-
-    private static BaseClickhouseData baseClickhouseData = new BaseClickhouseData();
-
-    @Test
-    public void testReadFqdnFromCk(){
-
-
-        HashMap<Integer, HashMap<String, ArrayList<BaseDocument>>> newData =
-                baseClickhouseData.baseDocFromCk(ReadClickhouseData::getVertexFqdnSql,
-                        ReadClickhouseData::getVertexFqdnDocument);
-        printMap(newData);
-
-    }
-
-    @Test
-    public void testReadFqdnLocIpFromCk(){
-        HashMap<Integer, HashMap<String, ArrayList<BaseEdgeDocument>>> map =
-                baseClickhouseData.baseDocFromCk(ReadClickhouseData::getRelationshipFqdnAddressIpSql,
-                        ReadClickhouseData::getRelationFqdnAddressIpDocument);
-
-        printMap(map);
-    }
-
-
-    private<T extends BaseDocument> void printMap(HashMap<Integer, HashMap<String, ArrayList<T>>> newData){
-        HashMap<String, ArrayList<T>> map = newData.get(1);
-        Set<String> strings = map.keySet();
-        for (String key:strings){
-            ArrayList<T> baseDocuments = map.get(key);
-            System.out.println(baseDocuments.get(0));
-        }
-    }
-}

IP-learning-graph/src/test/java/cn/ac/iie/readHistoryDataTest.java

@@ -6,5 +6,6 @@ import com.arangodb.entity.BaseEdgeDocument;
 public class readHistoryDataTest {
     public static void main(String[] args) {
         BaseArangoData baseArangoData = new BaseArangoData();
+
     }
 }

ip-learning-spark/pom.xml

@@ -63,18 +63,6 @@
         <version>6.6.3</version>
     </dependency>
 
-    <dependency>
-        <groupId>com.arangodb</groupId>
-        <artifactId>velocypack-module-jdk8</artifactId>
-        <version>1.1.0</version>
-    </dependency>
-
-    <dependency>
-        <groupId>com.arangodb</groupId>
-        <artifactId>velocypack-module-scala_2.11</artifactId>
-        <version>1.2.0</version>
-    </dependency>
-
     <dependency>
         <groupId>org.scala-lang</groupId>
         <artifactId>scala-library</artifactId>
@@ -87,11 +75,6 @@
         <version>3.2.0</version>
     </dependency>
 
-    <dependency>
-        <groupId>org.scala-lang.modules</groupId>
-        <artifactId>scala-xml_2.11</artifactId>
-        <version>1.0.4</version>
-    </dependency>
 
     <dependency>
         <groupId>org.scala-tools</groupId>

ip-learning-spark/src/main/java/cn/ac/iie/dao/BaseArangoData.java

@@ -23,9 +23,8 @@ public class BaseArangoData {
 
     private ExecutorThreadPool threadPool = ExecutorThreadPool.getInstance();
 
-    public <T extends BaseDocument> void readHistoryData(String table,
+    public <T extends BaseDocument> ConcurrentHashMap<Integer, ConcurrentHashMap<String, T>> readHistoryData(String table, Class<T> type) {
-                                                  ConcurrentHashMap<Integer, ConcurrentHashMap<String, T>> historyMap,
+        ConcurrentHashMap<Integer, ConcurrentHashMap<String, T>> historyMap = new ConcurrentHashMap<>();
-                                                  Class<T> type) {
         try {
             LOG.warn("开始更新" + table);
             long start = System.currentTimeMillis();
@@ -33,10 +32,8 @@ public class BaseArangoData {
                 historyMap.put(i, new ConcurrentHashMap<>());
             }
             CountDownLatch countDownLatch = new CountDownLatch(ApplicationConfig.THREAD_POOL_NUMBER());
-//            long[] timeRange = getTimeRange(table);
             Long countTotal = getCountTotal(table);
             for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER(); i++) {
-//                String sql = getQuerySql(timeRange, i, table);
                 String sql = getQuerySql(countTotal, i, table);
                 ReadHistoryArangoData<T> readHistoryArangoData = new ReadHistoryArangoData<>(arangoDBConnect, sql, historyMap, type, table, countDownLatch);
                 threadPool.executor(readHistoryArangoData);
@@ -47,6 +44,7 @@ public class BaseArangoData {
         } catch (Exception e) {
             e.printStackTrace();
         }
+        return historyMap;
     }
 
     private Long getCountTotal(String table){
@@ -62,14 +60,18 @@ public class BaseArangoData {
             LOG.error(sql +"执行异常");
         }
         long last = System.currentTimeMillis();
-        LOG.info(sql+" 结果："+cnt+" 执行时间："+(last-start));
+        LOG.warn(sql+" 结果："+cnt+" 执行时间："+(last-start));
         return cnt;
     }
 
     private String getQuerySql(Long cnt,int threadNumber, String table){
         long sepNum = cnt / ApplicationConfig.THREAD_POOL_NUMBER() + 1;
         long offsetNum = threadNumber * sepNum;
+        if (sepNum >= ApplicationConfig.ARANGODB_READ_LIMIT() * 10000){
+            sepNum = ApplicationConfig.ARANGODB_READ_LIMIT() * 10000;
+        }
         return "FOR doc IN " + table + " limit "+offsetNum+","+sepNum+" RETURN doc";
     }
 
+
 }

ip-learning-spark/src/main/java/cn/ac/iie/service/read/ReadHistoryArangoData.java

@@ -19,12 +19,13 @@ import java.util.concurrent.CountDownLatch;
  * @author wlh
  * 多线程全量读取arangoDb历史数据，封装到map
  */
+@SuppressWarnings("unchecked")
 public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
     public static long currentHour = System.currentTimeMillis() / (60 * 60 * 1000) * 60 * 60;
     private static final Logger LOG = LoggerFactory.getLogger(ReadHistoryArangoData.class);
-    static final Integer RECENT_COUNT_HOUR = ApplicationConfig.RECENT_COUNT_HOUR();
+    private static final Integer RECENT_COUNT_HOUR = ApplicationConfig.RECENT_COUNT_HOUR();
 
-    public static final HashSet<String> PROTOCOL_SET;
+    private static final HashSet<String> PROTOCOL_SET;
 
     static {
         PROTOCOL_SET = new HashSet<>();
@@ -69,9 +70,6 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
                             updateProtocolDocument(doc);
                             deleteDistinctClientIpByTime(doc);
                             break;
-                        case "R_VISIT_IP2FQDN":
-                            updateProtocolDocument(doc);
-                            break;
                         default:
                     }
                     int hashCode = Math.abs(key.hashCode()) % ApplicationConfig.THREAD_POOL_NUMBER();
@@ -95,7 +93,7 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
             for (String protocol : PROTOCOL_SET) {
                 String protocolRecent = protocol + "_CNT_RECENT";
                 ArrayList<Long> cntRecent = (ArrayList<Long>) doc.getAttribute(protocolRecent);
-                Long[] cntRecentsSrc = cntRecent.toArray(new Long[cntRecent.size()]);
+                Long[] cntRecentsSrc = cntRecent.toArray(new Long[0]);
                 Long[] cntRecentsDst = new Long[RECENT_COUNT_HOUR];
                 System.arraycopy(cntRecentsSrc, 0, cntRecentsDst, 1, cntRecentsSrc.length - 1);
                 cntRecentsDst[0] = 0L;
@@ -107,6 +105,11 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
     private void deleteDistinctClientIpByTime(T doc) {
         ArrayList<String> distCip = (ArrayList<String>) doc.getAttribute("DIST_CIP");
         ArrayList<Long> distCipTs = (ArrayList<Long>) doc.getAttribute("DIST_CIP_TS");
+        if (distCip == null || distCip.isEmpty()){
+            doc.updateAttribute("DIST_CIP", new String[0]);
+            doc.updateAttribute("DIST_CIP_TS", new long[0]);
+            return;
+        }
         distCipTs.add(currentHour - RECENT_COUNT_HOUR * 3600);
         Collections.sort(distCipTs);
         int index = distCipTs.indexOf(currentHour - RECENT_COUNT_HOUR * 3600);

ip-learning-spark/src/main/java/cn/ac/iie/utils/ArangoDBConnect.java

@@ -27,14 +27,27 @@ public class ArangoDBConnect {
     }
 
     private static void getArangoDatabase(){
-        arangoDB = new ArangoDB.Builder()
+        ArangoDB.Builder host = getArangoHost();
+        arangoDB = host
                 .maxConnections(ApplicationConfig.THREAD_POOL_NUMBER())
-                .host(ApplicationConfig.ARANGODB_HOST(), ApplicationConfig.ARANGODB_PORT())
+                .acquireHostList(true)
                 .user(ApplicationConfig.ARANGODB_USER())
                 .password(ApplicationConfig.ARANGODB_PASSWORD())
                 .build();
     }
 
+    private static ArangoDB.Builder getArangoHost(){
+        String hostList = ApplicationConfig.ARANGODB_HOST();
+        String[] split = hostList.split(",");
+        ArangoDB.Builder host = new ArangoDB.Builder();
+        for (String hostStr : split) {
+            host.host(hostStr, ApplicationConfig.ARANGODB_PORT());
+            LOG.warn("arangoDB host {} 已添加",hostStr);
+        }
+        LOG.warn("获取arangoDB host成功");
+        return host;
+    }
+
     public static synchronized ArangoDBConnect getInstance(){
         if (null == conn){
             conn = new ArangoDBConnect();
@@ -70,26 +83,6 @@ public class ArangoDBConnect {
         }
     }
 
-    @Deprecated
-    public <T> void insertAndUpdate(ArrayList<T> docInsert,ArrayList<T> docUpdate,String collectionName){
-        ArangoDatabase database = getDatabase();
-        try {
-            ArangoCollection collection = database.collection(collectionName);
-            if (!docInsert.isEmpty()){
-                collection.importDocuments(docInsert);
-            }
-            if (!docUpdate.isEmpty()){
-                collection.replaceDocuments(docUpdate);
-            }
-        }catch (Exception e){
-            System.out.println("更新失败");
-            e.printStackTrace();
-        }finally {
-            docInsert.clear();
-            docInsert.clear();
-        }
-    }
-
     public <T> void overwrite(ArrayList<T> docOverwrite,String collectionName){
         ArangoDatabase database = getDatabase();
         try {
@@ -101,11 +94,11 @@ public class ArangoDBConnect {
                 MultiDocumentEntity<DocumentCreateEntity<T>> documentCreateEntityMultiDocumentEntity = collection.insertDocuments(docOverwrite, documentCreateOptions);
                 Collection<ErrorEntity> errors = documentCreateEntityMultiDocumentEntity.getErrors();
                 for (ErrorEntity errorEntity:errors){
-                    LOG.warn("写入arangoDB异常："+errorEntity.getErrorMessage());
+                    LOG.debug("写入arangoDB异常："+errorEntity.getErrorMessage());
                 }
             }
         }catch (Exception e){
-            System.out.println("更新失败："+e.toString());
+            LOG.error("更新arangoDB失败："+e.toString());
         }finally {
             docOverwrite.clear();
         }

ip-learning-spark/src/main/resources/application.properties

@@ -4,9 +4,10 @@ spark.executor.memory=4g
 spark.app.name=test
 spark.network.timeout=300s
 spark.serializer=org.apache.spark.serializer.KryoSerializer
+#spark.serializer=org.apache.spark.serializer.JavaSerializer
 master=local[*]
 #spark读取clickhouse配置
-spark.read.clickhouse.url=jdbc:clickhouse://192.168.44.67:8123/tsg_galaxy_v3
+spark.read.clickhouse.url=jdbc:clickhouse://192.168.44.10:8123/tsg_galaxy_v3
 spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver
 spark.read.clickhouse.user=default
 spark.read.clickhouse.password=ceiec2019
@@ -15,22 +16,22 @@ spark.read.clickhouse.fetchsize=10000
 spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME
 clickhouse.socket.timeout=300000
 #arangoDB配置
-arangoDB.host=192.168.40.182
+arangoDB.host=192.168.40.123,192.168.40.223,192.168.40.222
+#arangoDB.host=192.168.40.223
 arangoDB.port=8529
 arangoDB.user=upsert
-arangoDB.password=ceiec2018
+arangoDB.password=ceiec2019
-arangoDB.DB.name=ip-learning-test-0
+arangoDB.DB.name=tsg_galaxy_v3
-#arangoDB.DB.name=iplearn_media_domain
 arangoDB.ttl=3600
 
 thread.pool.number=10
 
 #读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围
 clickhouse.time.limit.type=1
-read.clickhouse.max.time=1603785961
+read.clickhouse.max.time=1600916194
-read.clickhouse.min.time=1603354682
+read.clickhouse.min.time=1599197648
 
-arangoDB.read.limit=1
+arangoDB.read.sepNum=10
 update.arango.batch=10000
 
 distinct.client.ip.num=10000

ip-learning-spark/src/main/scala/cn/ac/iie/config/ApplicationConfig.scala

@@ -36,7 +36,7 @@ object ApplicationConfig {
   val READ_CLICKHOUSE_MAX_TIME: Long = config.getLong("read.clickhouse.max.time")
   val READ_CLICKHOUSE_MIN_TIME: Long = config.getLong("read.clickhouse.min.time")
 
-  val ARANGODB_READ_LIMIT: Int = config.getInt("arangoDB.read.limit")
+  val ARANGODB_READ_LIMIT: Long = config.getLong("arangoDB.read.sepNum")
   val UPDATE_ARANGO_BATCH: Int = config.getInt("update.arango.batch")
   val RECENT_COUNT_HOUR: Int = config.getInt("recent.count.hour")
   val DISTINCT_CLIENT_IP_NUM: Int = config.getInt("distinct.client.ip.num")

ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala

@@ -11,7 +11,7 @@ object BaseClickhouseData {
   val currentHour: Long = System.currentTimeMillis / (60 * 60 * 1000) * 60 * 60
   private val timeLimit: (Long, Long) = getTimeLimit
 
-  private def initClickhouseData(sql:String): DataFrame  ={
+  private def initClickhouseData(sql:String): DataFrame ={
 
     val dataFrame: DataFrame = spark.read.format("jdbc")
       .option("url", ApplicationConfig.SPARK_READ_CLICKHOUSE_URL)
@@ -33,7 +33,7 @@ object BaseClickhouseData {
   }
 
   def loadConnectionDataFromCk(): Unit ={
-    val where = "common_recv_time >= " + timeLimit._2 + " AND common_recv_time < " + timeLimit._1
+    val where = "common_recv_time >= " + timeLimit._2 + " AND common_recv_time < " + timeLimit._1 + " AND common_schema_type != 'BASE'"
     val sql =
       s"""
         |(SELECT
@@ -70,50 +70,75 @@ object BaseClickhouseData {
     initClickhouseData(sql)
   }
 
-  /*
+  def getVertexFqdnDf: DataFrame ={
-  def getVertexIpDf: DataFrame ={
     loadConnectionDataFromCk()
     val sql =
       """
         |SELECT
-        | *
+        | FQDN,MAX( LAST_FOUND_TIME ) AS LAST_FOUND_TIME,MIN( FIRST_FOUND_TIME ) AS FIRST_FOUND_TIME
         |FROM
         | (
-        |  (
+        |  (SELECT
-        |   SELECT
+        |    ssl_sni AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME
-        |    common_client_ip AS IP,
-        |    MIN(common_recv_time) AS FIRST_FOUND_TIME,
-        |    MAX(common_recv_time) AS LAST_FOUND_TIME,
-        |    count(*) as SESSION_COUNT,
-        |    sum(common_c2s_byte_num) as BYTES_SUM,
-        |    'client' as ip_type
         |   FROM
         |    global_temp.dbtable
-        |   GROUP BY
+        |   WHERE
-        |    IP
+        |    common_schema_type = 'SSL' GROUP BY ssl_sni
         |  )
-        | UNION ALL
+        |  UNION ALL
-        |  (
+        |  (SELECT
-        |   SELECT
+        |    http_host AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME
-        |    common_server_ip AS IP,
-        |    MIN(common_recv_time) AS FIRST_FOUND_TIME,
-        |    MAX(common_recv_time) AS LAST_FOUND_TIME,
-        |    count(*) as SESSION_COUNT,
-        |    sum(common_s2c_byte_num) as BYTES_SUM,
-        |    'server' as ip_type
         |   FROM
         |    global_temp.dbtable
-        |   GROUP BY
+        |   WHERE
-        |    IP
+        |    common_schema_type = 'HTTP' GROUP BY http_host
         |  )
         | )
+        |GROUP BY
+        | FQDN
+        |HAVING
+        | FQDN != ''
       """.stripMargin
     LOG.warn(sql)
-    val vertexIpDf = spark.sql(sql)
+    val vertexFqdnDf = spark.sql(sql)
-    vertexIpDf.printSchema()
+    vertexFqdnDf.printSchema()
-    vertexIpDf
+    vertexFqdnDf
   }
 
+  def getVertexIpDf: DataFrame ={
+    loadConnectionDataFromCk()
+    val where = "common_recv_time >= " + timeLimit._2 + " AND common_recv_time < " + timeLimit._1
+    val sql =
+      s"""
+        |(SELECT * FROM
+        |((SELECT common_client_ip AS IP,MIN(common_end_time) AS FIRST_FOUND_TIME,
+        |MAX(common_end_time) AS LAST_FOUND_TIME,
+        |count(*) as SESSION_COUNT,
+        |SUM(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,
+        |groupUniqArray(2)(common_link_info_c2s)[2] as common_link_info,
+        |'client' as ip_type
+        |FROM tsg_galaxy_v3.connection_record_log
+        |where $where
+        |group by common_client_ip)
+        |UNION ALL
+        |(SELECT common_server_ip AS IP,
+        |MIN(common_end_time) AS FIRST_FOUND_TIME,
+        |MAX(common_end_time) AS LAST_FOUND_TIME,
+        |count(*) as SESSION_COUNT,
+        |SUM(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,
+        |groupUniqArray(2)(common_link_info_s2c)[2] as common_link_info,
+        |'server' as ip_type
+        |FROM tsg_galaxy_v3.connection_record_log
+        |where $where
+        |group by common_server_ip))) as dbtable
+      """.stripMargin
+    LOG.warn(sql)
+    val frame = initClickhouseData(sql)
+    frame.printSchema()
+    frame
+  }
+
+  /*
   def getRelationFqdnLocateIpDf: DataFrame ={
     loadConnectionDataFromCk()
     val sslSql =
@@ -160,62 +185,6 @@ object BaseClickhouseData {
   }
   */
 
-  def getVertexFqdnDf: DataFrame ={
-    val sql =
-      """
-        |(SELECT
-        | FQDN,MAX( LAST_FOUND_TIME ) AS LAST_FOUND_TIME,MIN( FIRST_FOUND_TIME ) AS FIRST_FOUND_TIME
-        |FROM
-        | ((SELECT
-        |    ssl_sni AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME
-        |   FROM tsg_galaxy_v3.connection_record_log
-        |   WHERE common_schema_type = 'SSL' GROUP BY ssl_sni
-        |  )UNION ALL
-        |  (SELECT
-        |    http_host AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME
-        |   FROM tsg_galaxy_v3.connection_record_log
-        |   WHERE common_schema_type = 'HTTP' GROUP BY http_host))
-        |GROUP BY FQDN HAVING FQDN != '') as dbtable
-      """.stripMargin
-    LOG.warn(sql)
-    val frame = initClickhouseData(sql)
-    frame.printSchema()
-    frame
-  }
-
-  def getVertexIpDf: DataFrame ={
-    val where = "common_recv_time >= " + timeLimit._2 + " AND common_recv_time < " + timeLimit._1
-    val sql =
-      s"""
-         |(SELECT * FROM
-         |((SELECT common_client_ip AS IP,MIN(common_recv_time) AS FIRST_FOUND_TIME,
-         |MAX(common_recv_time) AS LAST_FOUND_TIME,
-         |count(*) as SESSION_COUNT,
-         |SUM(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,
-         |groupUniqArray(2)(common_link_info_c2s)[2] as common_link_info,
-         |'client' as ip_type
-         |FROM tsg_galaxy_v3.connection_record_log
-         |where $where
-         |group by common_client_ip)
-         |UNION ALL
-         |(SELECT common_server_ip AS IP,
-         |MIN(common_recv_time) AS FIRST_FOUND_TIME,
-         |MAX(common_recv_time) AS LAST_FOUND_TIME,
-         |count(*) as SESSION_COUNT,
-         |SUM(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,
-         |groupUniqArray(2)(common_link_info_s2c)[2] as common_link_info,
-         |'server' as ip_type
-         |FROM tsg_galaxy_v3.connection_record_log
-         |where $where
-         |group by common_server_ip))) as dbtable
-      """.stripMargin
-    LOG.warn(sql)
-    val frame = initClickhouseData(sql)
-    frame.printSchema()
-    frame
-  }
-
-
   def getRelationFqdnLocateIpDf: DataFrame ={
     val where = "common_recv_time >= " + timeLimit._2 + " AND common_recv_time < " + timeLimit._1
     val sql =
@@ -270,10 +239,10 @@ object BaseClickhouseData {
       """.stripMargin
     val sql =
       s"""
-         |(
+        |(
-         |SELECT common_subscriber_id,MAX(common_recv_time) as LAST_FOUND_TIME,MIN(common_recv_time) as FIRST_FOUND_TIME FROM radius_record_log
+        |SELECT common_subscriber_id,MAX(common_recv_time) as LAST_FOUND_TIME,MIN(common_recv_time) as FIRST_FOUND_TIME FROM radius_record_log
-         |WHERE $where GROUP BY common_subscriber_id
+        |WHERE $where GROUP BY common_subscriber_id
-         |)as dbtable
+        |)as dbtable
       """.stripMargin
     LOG.warn(sql)
     val frame = initClickhouseData(sql)
@@ -291,9 +260,9 @@ object BaseClickhouseData {
       """.stripMargin
     val sql =
       s"""
-         |(
+        |(
-         |SELECT DISTINCT radius_framed_ip,common_recv_time as LAST_FOUND_TIME FROM radius_record_log WHERE $where
+        |SELECT DISTINCT radius_framed_ip,common_recv_time as LAST_FOUND_TIME FROM radius_record_log WHERE $where
-         |)as dbtable
+        |)as dbtable
       """.stripMargin
     LOG.warn(sql)
     val frame = initClickhouseData(sql)
@@ -301,7 +270,6 @@ object BaseClickhouseData {
     frame
   }
 
-
   private def getTimeLimit: (Long,Long) ={
     var maxTime = 0L
     var minTime = 0L

ip-learning-spark/src/main/scala/cn/ac/iie/service/transform/MergeDataFrame.scala

@@ -4,36 +4,34 @@ import java.util.regex.Pattern
 
 import cn.ac.iie.config.ApplicationConfig
 import cn.ac.iie.dao.BaseClickhouseData
-import cn.ac.iie.spark.ArangoSpark
 import cn.ac.iie.spark.partition.CustomPartitioner
-import cn.ac.iie.spark.rdd.ReadOptions
-import com.arangodb.entity.{BaseDocument, BaseEdgeDocument}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.functions._
 import org.slf4j.LoggerFactory
-import cn.ac.iie.utils.SparkSessionUtil._
 
 object MergeDataFrame {
   private val LOG = LoggerFactory.getLogger(MergeDataFrame.getClass)
   private val pattern = Pattern.compile("^[\\d]*$")
-  private val options = ReadOptions(ApplicationConfig.ARANGODB_DB_NAME)
 
-  def mergeVertexFqdn(): RDD[(String, (Option[BaseDocument], Option[Row]))] ={
+  def mergeVertexFqdn(): RDD[Row] ={
-    val fqdnAccmu = getLongAccumulator("FQDN Accumulator")
+    BaseClickhouseData.getVertexFqdnDf
-    val fqdnRddRow = BaseClickhouseData.getVertexFqdnDf
+      .rdd.filter(row => isDomain(row.getAs[String](0))).map(row => (row.get(0),row))
-      .rdd.filter(row => isDomain(row.getAs[String](0))).map(row => {
+      .partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)).values
-      fqdnAccmu.add(1)
-      (row.getAs[String]("FQDN"), row)
-    }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS))
-    fqdnRddRow.cache()
-    val fqdnRddDoc = ArangoSpark.load[BaseDocument](sparkContext,"FQDN",options)
-
-    fqdnRddDoc.map(doc => (doc.getKey, doc)).fullOuterJoin(fqdnRddRow)
   }
 
-  def mergeVertexIp(): RDD[(String, (Option[BaseDocument], Option[Row]))]={
+  def mergeVertexFrameIp: RDD[Row] ={
-    val ipAccum = getLongAccumulator("IP Accumulator")
+    val values = BaseClickhouseData.getVertexFramedIpDf
+      .repartition(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)
+      .rdd.map(row => {
+      val ip = row.getAs[String]("radius_framed_ip")
+      (ip, row)
+    }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)).values
+    LOG.warn(s"读取R_LOCATE_SUBSCRIBER2IP clickhouse成功，共：${values.count()} 条")
+    values
+  }
+
+  def mergeVertexIp(): RDD[Row]={
     val vertexIpDf = BaseClickhouseData.getVertexIpDf
     val frame = vertexIpDf.groupBy("IP").agg(
       min("FIRST_FOUND_TIME").alias("FIRST_FOUND_TIME"),
@@ -43,19 +41,15 @@ object MergeDataFrame {
       collect_list("ip_type").alias("ip_type_list"),
       last("common_link_info").alias("common_link_info")
     )
-    val ipRddRow = frame.rdd.map(row => {
+    val values = frame.rdd.map(row => (row.get(0), row))
-      ipAccum.add(1)
+      .partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)).values
-      (row.getAs[String]("IP"), row)
+    values
-    }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS))
-    val ipRddDoc = ArangoSpark.load[BaseDocument](sparkContext,"IP",options)
-
-    ipRddDoc.map(doc => (doc.getKey, doc)).fullOuterJoin(ipRddRow)
-
   }
 
-  def mergeRelationFqdnLocateIp(): RDD[(String, (Option[BaseEdgeDocument], Option[Row]))] ={
+  def mergeRelationFqdnLocateIp(): RDD[Row] ={
-    val fqdnLocIpAccum = getLongAccumulator("R_LOCATE_FQDN2IP Accumulator")
+    val frame = BaseClickhouseData.getRelationFqdnLocateIpDf
-    val frame = BaseClickhouseData.getRelationFqdnLocateIpDf.filter(row => isDomain(row.getAs[String]("FQDN")))
+      .repartition(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)
+      .filter(row => isDomain(row.getAs[String]("FQDN")))
       .groupBy("FQDN", "common_server_ip")
       .agg(
         min("FIRST_FOUND_TIME").alias("FIRST_FOUND_TIME"),
@@ -64,60 +58,38 @@ object MergeDataFrame {
         collect_list("schema_type").alias("schema_type_list"),
         collect_set("DIST_CIP_RECENT").alias("DIST_CIP_RECENT")
       )
-    val fqdnLocIpRddRow = frame.rdd.map(row => {
+    val values = frame.rdd.map(row => {
       val fqdn = row.getAs[String]("FQDN")
       val serverIp = row.getAs[String]("common_server_ip")
       val key = fqdn.concat("-" + serverIp)
-      fqdnLocIpAccum.add(1)
       (key, row)
-    }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS))
+    }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)).values
-    val fqdnLocIpRddDoc = ArangoSpark.load[BaseEdgeDocument](sparkContext,"R_LOCATE_FQDN2IP",options)
+    LOG.warn(s"读取R_LOCATE_FQDN2IP clickhouse成功，共：${values.count()} 条")
-
+    values
-    fqdnLocIpRddDoc.map(doc => (doc.getKey, doc)).fullOuterJoin(fqdnLocIpRddRow)
 
   }
 
-  def mergeRelationSubidLocateIp(): RDD[(String, (Option[BaseEdgeDocument], Option[Row]))] ={
+  def mergeRelationSubidLocateIp(): RDD[Row] ={
-    val subidLocIpAccum = getLongAccumulator("R_LOCATE_SUBSCRIBER2IP Accumulator")
+    val values = BaseClickhouseData.getRelationSubidLocateIpDf
-    val subidLocIpRddRow = BaseClickhouseData.getRelationSubidLocateIpDf
       .repartition(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)
       .rdd.map(row => {
       val commonSubscriberId = row.getAs[String]("common_subscriber_id")
       val ip = row.getAs[String]("radius_framed_ip")
       val key = commonSubscriberId.concat("-" + ip)
-      subidLocIpAccum.add(1)
       (key, row)
-    }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS))
+    }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)).values
-    val subidLocIpRddDoc = ArangoSpark.load[BaseEdgeDocument](sparkContext,"R_LOCATE_SUBSCRIBER2IP",options)
+    LOG.warn(s"读取R_LOCATE_SUBSCRIBER2IP clickhouse成功，共：${values.count()} 条")
-
+    values
-    subidLocIpRddDoc.map(doc => (doc.getKey, doc)).fullOuterJoin(subidLocIpRddRow)
   }
 
-  def mergeVertexSubid(): RDD[(String, (Option[BaseDocument], Option[Row]))] ={
+  def mergeVertexSubid(): RDD[Row] ={
-    val subidAccum = getLongAccumulator("SUBSCRIBER Accumulator")
+    val values = BaseClickhouseData.getVertexSubidDf
-    val subidRddRow = BaseClickhouseData.getVertexSubidDf
       .repartition(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)
       .rdd.map(row => {
       val commonSubscriberId = row.getAs[String]("common_subscriber_id")
-      subidAccum.add(1)
       (commonSubscriberId, row)
-    }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS))
-
-    val subidRddDoc = ArangoSpark.load[BaseDocument](sparkContext,"SUBSCRIBER",options)
-
-    subidRddDoc.map(doc => (doc.getKey, doc)).fullOuterJoin(subidRddRow)
-
-  }
-
-  def mergeVertexFrameIp: RDD[Row] ={
-    val framedIpAccum = getLongAccumulator("framed ip Accumulator")
-    val values = BaseClickhouseData.getVertexFramedIpDf
-      .repartition(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)
-      .rdd.map(row => {
-      val ip = row.getAs[String]("radius_framed_ip")
-      framedIpAccum.add(1)
-      (ip, row)
     }).partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)).values
+    LOG.warn(s"读取SUBSCRIBER clickhouse成功，共：${values.count()} 条")
     values
   }
 
@@ -147,7 +119,6 @@ object MergeDataFrame {
         LOG.error("解析域名 " + fqdn + " 失败：\n" + e.toString)
     }
     false
-
   }
 
 }

ip-learning-spark/src/main/scala/cn/ac/iie/service/update/UpdateDocHandler.scala

@@ -1,8 +1,7 @@
 package cn.ac.iie.service.update
 
 
-import java.util
+import java.lang
-import scala.collection.JavaConversions._
 
 import cn.ac.iie.config.ApplicationConfig
 import cn.ac.iie.service.read.ReadHistoryArangoData
@@ -15,20 +14,16 @@ object UpdateDocHandler {
   val PROTOCOL_SET: Set[String] = Set("HTTP","TLS","DNS")
 
   def updateMaxAttribute(hisDoc: BaseDocument,newAttribute:Long,attributeName:String): Unit ={
-    if(hisDoc.getProperties.containsKey(attributeName)){
+    var hisAttritube = hisDoc.getAttribute(attributeName).toString.toLong
-      var hisAttritube = hisDoc.getAttribute(attributeName).toString.toLong
+    if (newAttribute > hisAttritube){
-      if (newAttribute > hisAttritube){
+      hisAttritube = newAttribute
-        hisAttritube = newAttribute
-      }
-      hisDoc.addAttribute(attributeName,hisAttritube)
     }
+    hisDoc.addAttribute(attributeName,hisAttritube)
   }
 
   def updateSumAttribute(hisDoc: BaseDocument,newAttribute:Long,attributeName:String): Unit ={
-    if (hisDoc.getProperties.containsKey(attributeName)){
+    val hisAttritube = hisDoc.getAttribute(attributeName).toString.toLong
-      val hisAttritube = hisDoc.getAttribute(attributeName).toString.toLong
+    hisDoc.addAttribute(attributeName,newAttribute+hisAttritube)
-      hisDoc.addAttribute(attributeName,newAttribute+hisAttritube)
-    }
   }
 
   def replaceAttribute(hisDoc: BaseDocument,newAttribute:String,attributeName:String): Unit ={
@@ -71,21 +66,19 @@ object UpdateDocHandler {
   }
 
   def updateProtocolAttritube(hisDoc:BaseEdgeDocument, protocolMap: Map[String, Long]): Unit ={
-    if (hisDoc.getProperties.containsKey("PROTOCOL_TYPE")){
+    var protocolType = hisDoc.getAttribute("PROTOCOL_TYPE").toString
-      var protocolType = hisDoc.getAttribute("PROTOCOL_TYPE").toString
+    protocolMap.foreach(t => {
-      protocolMap.foreach((t: (String, Long)) => {
+      if (t._2 > 0 && !protocolType.contains(t._1)){
-        if (t._2 > 0 && !protocolType.contains(t._1)){
+        protocolType = protocolType.concat(","+ t._1)
-          protocolType = protocolType.concat(","+ t._1)
+      }
-        }
+      val cntTotalName = t._1.concat("_CNT_TOTAL")
-        val cntTotalName = t._1.concat("_CNT_TOTAL")
+      val cntRecentName = t._1.concat("_CNT_RECENT")
-        val cntRecentName = t._1.concat("_CNT_RECENT")
+      val cntRecent: Array[lang.Long] = hisDoc.getAttribute(cntRecentName).asInstanceOf[Array[java.lang.Long]]
-        val cntRecent = hisDoc.getAttribute(cntRecentName).asInstanceOf[Array[Long]]
+      cntRecent.update(0,t._2)
-        cntRecent.update(0,t._2)
+      updateSumAttribute(hisDoc,t._2,cntTotalName)
-        updateSumAttribute(hisDoc,t._2,cntTotalName)
+      hisDoc.addAttribute(cntRecentName,cntRecent)
-        hisDoc.addAttribute(cntRecentName,cntRecent)
+    })
-      })
+    hisDoc.addAttribute("PROTOCOL_TYPE",protocolType)
-      hisDoc.addAttribute("PROTOCOL_TYPE",protocolType)
-    }
   }
 
   def putProtocolAttritube(doc:BaseEdgeDocument, protocolMap: Map[String, Long]): Unit ={
@@ -104,30 +97,15 @@ object UpdateDocHandler {
     doc.addAttribute("PROTOCOL_TYPE",protocolTypeBuilder.toString().replaceFirst(",",""))
   }
 
-  def updateProtocolDocument(doc: BaseEdgeDocument): Unit = {
-    if (doc.getProperties.containsKey("PROTOCOL_TYPE")) {
-      for (protocol <- PROTOCOL_SET) {
-        val protocolRecent = protocol + "_CNT_RECENT"
-        val cntRecent: util.ArrayList[Long] = doc.getAttribute(protocolRecent).asInstanceOf[util.ArrayList[Long]]
-        val cntRecentsSrc = cntRecent.toArray().map(_.toString.toLong)
-        val cntRecentsDst = new Array[Long](24)
-        System.arraycopy(cntRecentsSrc, 0, cntRecentsDst, 1, cntRecentsSrc.length - 1)
-        cntRecentsDst(0) = 0L
-        doc.addAttribute(protocolRecent, cntRecentsDst)
-      }
-    }
-  }
-
   def mergeDistinctIp(distCipRecent:ofRef[String]): Array[String] ={
-    distCipRecent.flatMap(str => {
+     distCipRecent.flatMap(str => {
-      str.replaceAll("\\[","")
+       str.replaceAll("\\[","")
-        .replaceAll("\\]","")
+       .replaceAll("\\]","")
-        .replaceAll("\\'","")
+       .replaceAll("\\'","")
-        .split(",")
+         .split(",")
-    }).distinct.take(ApplicationConfig.DISTINCT_CLIENT_IP_NUM).toArray
+     }).distinct.take(ApplicationConfig.DISTINCT_CLIENT_IP_NUM).toArray
   }
 
-
   def putDistinctIp(doc:BaseEdgeDocument,newDistinctIp:Array[String]): Unit ={
     val map = newDistinctIp.map(ip => {
       (ip, ReadHistoryArangoData.currentHour)
@@ -137,19 +115,17 @@ object UpdateDocHandler {
   }
 
   def updateDistinctIp(hisDoc:BaseEdgeDocument,newDistinctIp:Array[String]): Unit ={
-    if (hisDoc.getProperties.containsKey("DIST_CIP") && hisDoc.getProperties.containsKey("DIST_CIP_TS")){
+    val hisDistCip = hisDoc.getAttribute("DIST_CIP").asInstanceOf[Array[String]]
-      val hisDistCip = hisDoc.getAttribute("DIST_CIP").asInstanceOf[util.ArrayList[String]]
+    val hisDistCipTs = hisDoc.getAttribute("DIST_CIP_TS").asInstanceOf[Array[Long]]
-      val hisDistCipTs = hisDoc.getAttribute("DIST_CIP_TS").asInstanceOf[util.ArrayList[Long]]
+    if (hisDistCip.length == hisDistCipTs.length){
-      if (hisDistCip.length == hisDistCipTs.length){
+      val distCipToTsMap: Map[String, Long] = hisDistCip.zip(hisDistCipTs).toMap
-        val distCipToTsMap: Map[String, Long] = hisDistCip.zip(hisDistCipTs).toMap
+      val muDistCipToTsMap: mutable.Map[String, Long] = mutable.Map(distCipToTsMap.toSeq:_*)
-        val muDistCipToTsMap: mutable.Map[String, Long] = mutable.Map(distCipToTsMap.toSeq:_*)
+      newDistinctIp.foreach(cip => {
-        newDistinctIp.foreach(cip => {
+        muDistCipToTsMap.put(cip,ReadHistoryArangoData.currentHour)
-          muDistCipToTsMap.put(cip,ReadHistoryArangoData.currentHour)
+      })
-        })
+      val resultMap = muDistCipToTsMap.toList.sortBy(-_._2).take(ApplicationConfig.DISTINCT_CLIENT_IP_NUM).toMap
-        val resultMap = muDistCipToTsMap.toList.sortBy(-_._2).take(ApplicationConfig.DISTINCT_CLIENT_IP_NUM).toMap
+      hisDoc.addAttribute("DIST_CIP",resultMap.keys.toArray)
-        hisDoc.addAttribute("DIST_CIP",resultMap.keys.toArray)
+      hisDoc.addAttribute("DIST_CIP_TS",resultMap.values.toArray)
-        hisDoc.addAttribute("DIST_CIP_TS",resultMap.values.toArray)
-      }
     }
   }
 

ip-learning-spark/src/main/scala/cn/ac/iie/service/update/UpdateDocument.scala

@@ -1,12 +1,16 @@
 package cn.ac.iie.service.update
 
 import java.util
+import java.util.concurrent.ConcurrentHashMap
 
 import cn.ac.iie.config.ApplicationConfig
+import cn.ac.iie.dao.BaseArangoData
 import cn.ac.iie.service.transform.MergeDataFrame._
 import cn.ac.iie.service.update.UpdateDocHandler._
-import cn.ac.iie.utils.{ArangoDBConnect, SparkSessionUtil}
+import cn.ac.iie.utils.{ArangoDBConnect, ExecutorThreadPool, SparkSessionUtil}
+import cn.ac.iie.utils.SparkSessionUtil.spark
 import com.arangodb.entity.{BaseDocument, BaseEdgeDocument}
+import org.apache.spark.TaskContext
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.Row
 import org.slf4j.LoggerFactory
@@ -14,63 +18,25 @@ import org.slf4j.LoggerFactory
 import scala.collection.mutable.WrappedArray.ofRef
 
 object UpdateDocument {
+  private val pool = ExecutorThreadPool.getInstance
   private val arangoManger: ArangoDBConnect = ArangoDBConnect.getInstance()
   private val LOG = LoggerFactory.getLogger(UpdateDocument.getClass)
+  private val baseArangoData = new BaseArangoData()
 
   def update(): Unit = {
     try {
-      updateDocument("FQDN", getVertexFqdnRow, mergeVertexFqdn)
+//      updateDocument("FQDN", getVertexFqdnRow, classOf[BaseDocument], mergeVertexFqdn)
-
+//      updateDocument("IP", getVertexIpRow, classOf[BaseDocument], mergeVertexIp)
-      updateDocument("SUBSCRIBER",getVertexSubidRow,mergeVertexSubid)
+      updateDocument("R_LOCATE_FQDN2IP", getRelationFqdnLocateIpRow, classOf[BaseEdgeDocument], mergeRelationFqdnLocateIp)
-
+      updateDocument("SUBSCRIBER",getVertexSubidRow,classOf[BaseDocument],mergeVertexSubid)
       insertFrameIp()
-
+      updateDocument("R_LOCATE_SUBSCRIBER2IP",getRelationSubidLocateIpRow,classOf[BaseEdgeDocument],mergeRelationSubidLocateIp)
-      updateDocument("R_LOCATE_SUBSCRIBER2IP",getRelationSubidLocateIpRow,mergeRelationSubidLocateIp)
-
-      updateDocument("R_LOCATE_FQDN2IP", getRelationFqdnLocateIpRow, mergeRelationFqdnLocateIp)
-
-      updateDocument("IP", getVertexIpRow, mergeVertexIp)
-
     } catch {
       case e: Exception => e.printStackTrace()
     } finally {
+      pool.shutdown()
       arangoManger.clean()
       SparkSessionUtil.closeSpark()
-      System.exit(0)
-    }
-  }
-
-  private def updateDocument[T <: BaseDocument](collName: String,
-                                                getDocumentRow: ((String, (Option[T], Option[Row]))) => T,
-                                                getJoinRdd: () => RDD[(String, (Option[T], Option[Row]))]
-                                               ): Unit = {
-    try {
-      val start = System.currentTimeMillis()
-      val joinRdd = getJoinRdd()
-      joinRdd.foreachPartition(iter => {
-        val resultDocumentList = new util.ArrayList[T]
-        var i = 0
-        iter.foreach(row => {
-          val document = getDocumentRow(row)
-          if (document != null){
-            resultDocumentList.add(document)
-          }
-          i += 1
-          if (i >= ApplicationConfig.UPDATE_ARANGO_BATCH) {
-            arangoManger.overwrite(resultDocumentList, collName)
-            LOG.warn(s"更新:$collName" + i)
-            i = 0
-          }
-        })
-        if (i != 0) {
-          arangoManger.overwrite(resultDocumentList, collName)
-          LOG.warn(s"更新$collName:" + i)
-        }
-      })
-      val last = System.currentTimeMillis()
-      LOG.warn(s"更新$collName 时间：${last - start}")
-    } catch {
-      case e: Exception => e.printStackTrace()
     }
   }
 
@@ -95,6 +61,86 @@ object UpdateDocument {
     })
   }
 
+  private def updateDocument[T <: BaseDocument](collName: String,
+                                                getDocumentRow: (Row, ConcurrentHashMap[String, T]) => T,
+                                                clazz: Class[T],
+                                                getNewDataRdd: () => RDD[Row]
+                                               ): Unit = {
+    val historyMap = baseArangoData.readHistoryData(collName, clazz)
+    val hisBc = spark.sparkContext.broadcast(historyMap)
+    try {
+      val start = System.currentTimeMillis()
+      val newDataRdd = getNewDataRdd()
+      newDataRdd.foreachPartition(iter => {
+        val partitionId: Int = TaskContext.get.partitionId
+        val dictionaryMap: ConcurrentHashMap[String, T] = hisBc.value.get(partitionId)
+        val resultDocumentList = new util.ArrayList[T]
+        var i = 0
+        iter.foreach(row => {
+          val document = getDocumentRow(row, dictionaryMap)
+          resultDocumentList.add(document)
+          i += 1
+          if (i >= ApplicationConfig.UPDATE_ARANGO_BATCH) {
+            arangoManger.overwrite(resultDocumentList, collName)
+            LOG.warn(s"更新:$collName" + i)
+            i = 0
+          }
+        })
+        if (i != 0) {
+          arangoManger.overwrite(resultDocumentList, collName)
+          LOG.warn(s"更新$collName:" + i)
+        }
+      })
+      val last = System.currentTimeMillis()
+      LOG.warn(s"更新$collName 时间：${last - start}")
+    } catch {
+      case e: Exception => e.printStackTrace()
+    } finally {
+      hisBc.destroy()
+    }
+  }
+
+  private def getVertexFqdnRow(row: Row, dictionaryMap: ConcurrentHashMap[String, BaseDocument]): BaseDocument = {
+    val fqdn = row.getAs[String]("FQDN")
+    val lastFoundTime = row.getAs[Long]("LAST_FOUND_TIME")
+    val firstFoundTime = row.getAs[Long]("FIRST_FOUND_TIME")
+    var document: BaseDocument = dictionaryMap.getOrDefault(fqdn, null)
+    if (document != null) {
+      updateMaxAttribute(document, lastFoundTime, "LAST_FOUND_TIME")
+    } else {
+      document = new BaseDocument
+      document.setKey(fqdn)
+      document.addAttribute("FQDN_NAME", fqdn)
+      document.addAttribute("FIRST_FOUND_TIME", firstFoundTime)
+      document.addAttribute("LAST_FOUND_TIME", lastFoundTime)
+    }
+    document
+  }
+
+  private def getRelationSubidLocateIpRow(row: Row, dictionaryMap: ConcurrentHashMap[String, BaseEdgeDocument]): BaseEdgeDocument ={
+    val subId = row.getAs[String]("common_subscriber_id")
+    val ip = row.getAs[String]("radius_framed_ip")
+    val lastFoundTime = row.getAs[Long]("LAST_FOUND_TIME")
+    val firstFoundTime = row.getAs[Long]("FIRST_FOUND_TIME")
+
+    val key = subId.concat("-"+ip)
+    var document = dictionaryMap.getOrDefault(key,null)
+    if (document != null){
+      updateMaxAttribute(document,lastFoundTime,"LAST_FOUND_TIME")
+    } else {
+      document = new BaseEdgeDocument()
+      document.setKey(key)
+      document.setFrom("SUBSCRIBER/" + subId)
+      document.setTo("IP/" + ip)
+      document.addAttribute("SUBSCRIBER",subId)
+      document.addAttribute("IP",ip)
+      document.addAttribute("FIRST_FOUND_TIME",firstFoundTime)
+      document.addAttribute("LAST_FOUND_TIME",lastFoundTime)
+    }
+
+    document
+  }
+
   private def getVertexFrameipRow(row: Row): BaseDocument ={
     val ip = row.getAs[String]("radius_framed_ip")
     val document = new BaseDocument()
@@ -103,206 +149,86 @@ object UpdateDocument {
     document
   }
 
-  private def getRelationSubidLocateIpRow(joinRow: (String, (Option[BaseEdgeDocument], Option[Row]))): BaseEdgeDocument ={
+  private def getVertexSubidRow(row: Row, dictionaryMap: ConcurrentHashMap[String, BaseDocument]): BaseDocument ={
-
+    val subId = row.getAs[String]("common_subscriber_id")
-    val subidLocIpDocOpt = joinRow._2._1
+    val subLastFoundTime = row.getAs[Long]("LAST_FOUND_TIME")
-    var subidLocIpDoc = subidLocIpDocOpt match {
+    val subFirstFoundTime = row.getAs[Long]("FIRST_FOUND_TIME")
-      case Some(doc) => doc
+    var document = dictionaryMap.getOrDefault(subId,null)
-      case None => null
+    if (document != null){
+      updateMaxAttribute(document,subLastFoundTime,"LAST_FOUND_TIME")
+    } else {
+      document = new BaseDocument()
+      document.setKey(subId)
+      document.addAttribute("SUBSCRIBER",subId)
+      document.addAttribute("FIRST_FOUND_TIME",subFirstFoundTime)
+      document.addAttribute("LAST_FOUND_TIME",subLastFoundTime)
     }
 
-    val subidLocIpRowOpt = joinRow._2._2
+    document
-
-    val subidLocIpRow = subidLocIpRowOpt match {
-      case Some(r) => r
-      case None => null
-    }
-
-    if (subidLocIpRow != null){
-      val subId = subidLocIpRow.getAs[String]("common_subscriber_id")
-      val ip = subidLocIpRow.getAs[String]("radius_framed_ip")
-      val lastFoundTime = subidLocIpRow.getAs[Long]("LAST_FOUND_TIME")
-      val firstFoundTime = subidLocIpRow.getAs[Long]("FIRST_FOUND_TIME")
-
-      val key = subId.concat("-"+ip)
-      if (subidLocIpDoc != null){
-        updateMaxAttribute(subidLocIpDoc,lastFoundTime,"LAST_FOUND_TIME")
-      } else {
-        subidLocIpDoc = new BaseEdgeDocument()
-        subidLocIpDoc.setKey(key)
-        subidLocIpDoc.setFrom("SUBSCRIBER/" + subId)
-        subidLocIpDoc.setTo("IP/" + ip)
-        subidLocIpDoc.addAttribute("SUBSCRIBER",subId)
-        subidLocIpDoc.addAttribute("IP",ip)
-        subidLocIpDoc.addAttribute("FIRST_FOUND_TIME",firstFoundTime)
-        subidLocIpDoc.addAttribute("LAST_FOUND_TIME",lastFoundTime)
-      }
-    }
-    subidLocIpDoc
   }
 
-  private def getVertexSubidRow(joinRow: (String, (Option[BaseDocument], Option[Row]))): BaseDocument ={
+  private def getVertexIpRow(row: Row, dictionaryMap: ConcurrentHashMap[String, BaseDocument]): BaseDocument = {
-    val subidDocOpt = joinRow._2._1
+    val ip = row.getAs[String]("IP")
-    var subidDoc = subidDocOpt match {
+    val firstFoundTime = row.getAs[Long]("FIRST_FOUND_TIME")
-      case Some(doc) => doc
+    val lastFoundTime = row.getAs[Long]("LAST_FOUND_TIME")
-      case None => null
+    val sessionCountList = row.getAs[ofRef[AnyRef]]("SESSION_COUNT_LIST")
+    val bytesSumList = row.getAs[ofRef[AnyRef]]("BYTES_SUM_LIST")
+    val ipTypeList = row.getAs[ofRef[String]]("ip_type_list")
+    val linkInfo = row.getAs[String]("common_link_info")
+    val sepAttributeTuple = separateAttributeByIpType(ipTypeList, sessionCountList, bytesSumList)
+
+    var document = dictionaryMap.getOrDefault(ip, null)
+    if (document != null) {
+      updateMaxAttribute(document, lastFoundTime, "LAST_FOUND_TIME")
+      updateSumAttribute(document, sepAttributeTuple._1, "SERVER_SESSION_COUNT")
+      updateSumAttribute(document, sepAttributeTuple._2, "SERVER_BYTES_SUM")
+      updateSumAttribute(document, sepAttributeTuple._3, "CLIENT_SESSION_COUNT")
+      updateSumAttribute(document, sepAttributeTuple._4, "CLIENT_BYTES_SUM")
+      replaceAttribute(document,linkInfo,"COMMON_LINK_INFO")
+    } else {
+      document = new BaseDocument
+      document.setKey(ip)
+      document.addAttribute("IP", ip)
+      document.addAttribute("FIRST_FOUND_TIME", firstFoundTime)
+      document.addAttribute("LAST_FOUND_TIME", lastFoundTime)
+      document.addAttribute("SERVER_SESSION_COUNT", sepAttributeTuple._1)
+      document.addAttribute("SERVER_BYTES_SUM", sepAttributeTuple._2)
+      document.addAttribute("CLIENT_SESSION_COUNT", sepAttributeTuple._3)
+      document.addAttribute("CLIENT_BYTES_SUM", sepAttributeTuple._4)
+      document.addAttribute("COMMON_LINK_INFO", linkInfo)
     }
-
+    document
-    val subidRowOpt = joinRow._2._2
-
-    val subidRow = subidRowOpt match {
-      case Some(r) => r
-      case None => null
-    }
-
-    if (subidRow != null){
-      val subId = subidRow.getAs[String]("common_subscriber_id")
-      val subLastFoundTime = subidRow.getAs[Long]("LAST_FOUND_TIME")
-      val subFirstFoundTime = subidRow.getAs[Long]("FIRST_FOUND_TIME")
-      if (subidDoc != null){
-        updateMaxAttribute(subidDoc,subLastFoundTime,"LAST_FOUND_TIME")
-      } else {
-        subidDoc = new BaseDocument()
-        subidDoc.setKey(subId)
-        subidDoc.addAttribute("SUBSCRIBER",subId)
-        subidDoc.addAttribute("FIRST_FOUND_TIME",subFirstFoundTime)
-        subidDoc.addAttribute("LAST_FOUND_TIME",subLastFoundTime)
-      }
-    }
-
-    subidDoc
   }
 
-  private def getVertexFqdnRow(joinRow: (String, (Option[BaseDocument], Option[Row]))): BaseDocument = {
+  private def getRelationFqdnLocateIpRow(row: Row, dictionaryMap: ConcurrentHashMap[String, BaseEdgeDocument]): BaseEdgeDocument = {
-    val fqdnDocOpt = joinRow._2._1
+    val fqdn = row.getAs[String]("FQDN")
-    var fqdnDoc = fqdnDocOpt match {
+    val serverIp = row.getAs[String]("common_server_ip")
-      case Some(doc) => doc
+    val firstFoundTime = row.getAs[Long]("FIRST_FOUND_TIME")
-      case None => null
+    val lastFoundTime = row.getAs[Long]("LAST_FOUND_TIME")
+    val countTotalList = row.getAs[ofRef[AnyRef]]("COUNT_TOTAL_LIST")
+    val schemaTypeList = row.getAs[ofRef[AnyRef]]("schema_type_list")
+    val distCipRecent = row.getAs[ofRef[String]]("DIST_CIP_RECENT")
+
+    val sepAttritubeMap: Map[String, Long] = separateAttributeByProtocol(schemaTypeList, countTotalList)
+    val distinctIp: Array[String] = mergeDistinctIp(distCipRecent)
+
+    val key = fqdn.concat("-" + serverIp)
+    var document = dictionaryMap.getOrDefault(key, null)
+    if (document != null) {
+      updateMaxAttribute(document, lastFoundTime, "LAST_FOUND_TIME")
+      updateProtocolAttritube(document, sepAttritubeMap)
+      updateDistinctIp(document, distinctIp)
+    } else {
+      document = new BaseEdgeDocument()
+      document.setKey(key)
+      document.setFrom("FQDN/" + fqdn)
+      document.setTo("IP/" + serverIp)
+      document.addAttribute("FIRST_FOUND_TIME", firstFoundTime)
+      document.addAttribute("LAST_FOUND_TIME", lastFoundTime)
+      putProtocolAttritube(document, sepAttritubeMap)
+      putDistinctIp(document, distinctIp)
     }
-
+    document
-    val fqdnRowOpt = joinRow._2._2
-
-    val fqdnRow = fqdnRowOpt match {
-      case Some(r) => r
-      case None => null
-    }
-
-    if (fqdnRow != null){
-      val fqdn = fqdnRow.getAs[String]("FQDN")
-      val lastFoundTime = fqdnRow.getAs[Long]("LAST_FOUND_TIME")
-      val firstFoundTime = fqdnRow.getAs[Long]("FIRST_FOUND_TIME")
-      if (fqdnDoc != null) {
-        updateMaxAttribute(fqdnDoc, lastFoundTime, "LAST_FOUND_TIME")
-      } else {
-        fqdnDoc = new BaseDocument
-        fqdnDoc.setKey(fqdn)
-        fqdnDoc.addAttribute("FQDN_NAME", fqdn)
-        fqdnDoc.addAttribute("FIRST_FOUND_TIME", firstFoundTime)
-        fqdnDoc.addAttribute("LAST_FOUND_TIME", lastFoundTime)
-      }
-    }
-
-    fqdnDoc
-  }
-
-  private def getVertexIpRow(joinRow: (String, (Option[BaseDocument], Option[Row]))): BaseDocument = {
-    val ipDocOpt = joinRow._2._1
-    var ipDoc = ipDocOpt match {
-      case Some(doc) => doc
-      case None => null
-    }
-
-    val ipRowOpt = joinRow._2._2
-
-    val ipRow = ipRowOpt match {
-      case Some(r) => r
-      case None => null
-    }
-
-    if (ipRow != null){
-      val ip = ipRow.getAs[String]("IP")
-      val firstFoundTime = ipRow.getAs[Long]("FIRST_FOUND_TIME")
-      val lastFoundTime = ipRow.getAs[Long]("LAST_FOUND_TIME")
-      val sessionCountList = ipRow.getAs[ofRef[AnyRef]]("SESSION_COUNT_LIST")
-      val bytesSumList = ipRow.getAs[ofRef[AnyRef]]("BYTES_SUM_LIST")
-      val ipTypeList = ipRow.getAs[ofRef[String]]("ip_type_list")
-      val linkInfo = ipRow.getAs[String]("common_link_info")
-      val sepAttributeTuple = separateAttributeByIpType(ipTypeList, sessionCountList, bytesSumList)
-
-      if (ipDoc != null) {
-        updateMaxAttribute(ipDoc, lastFoundTime, "LAST_FOUND_TIME")
-        updateSumAttribute(ipDoc, sepAttributeTuple._1, "SERVER_SESSION_COUNT")
-        updateSumAttribute(ipDoc, sepAttributeTuple._2, "SERVER_BYTES_SUM")
-        updateSumAttribute(ipDoc, sepAttributeTuple._3, "CLIENT_SESSION_COUNT")
-        updateSumAttribute(ipDoc, sepAttributeTuple._4, "CLIENT_BYTES_SUM")
-        replaceAttribute(ipDoc,linkInfo,"COMMON_LINK_INFO")
-      } else {
-        ipDoc = new BaseDocument
-        ipDoc.setKey(ip)
-        ipDoc.addAttribute("IP", ip)
-        ipDoc.addAttribute("FIRST_FOUND_TIME", firstFoundTime)
-        ipDoc.addAttribute("LAST_FOUND_TIME", lastFoundTime)
-        ipDoc.addAttribute("SERVER_SESSION_COUNT", sepAttributeTuple._1)
-        ipDoc.addAttribute("SERVER_BYTES_SUM", sepAttributeTuple._2)
-        ipDoc.addAttribute("CLIENT_SESSION_COUNT", sepAttributeTuple._3)
-        ipDoc.addAttribute("CLIENT_BYTES_SUM", sepAttributeTuple._4)
-        ipDoc.addAttribute("COMMON_LINK_INFO", "")
-      }
-    }
-
-    ipDoc
-  }
-
-  private def getRelationFqdnLocateIpRow(joinRow: (String, (Option[BaseEdgeDocument], Option[Row]))): BaseEdgeDocument = {
-
-    val fqdnLocIpDocOpt = joinRow._2._1
-    var fqdnLocIpDoc = fqdnLocIpDocOpt match {
-      case Some(doc) => doc
-      case None => null
-    }
-
-    val fqdnLocIpRowOpt = joinRow._2._2
-
-    val fqdnLocIpRow = fqdnLocIpRowOpt match {
-      case Some(r) => r
-      case None => null
-    }
-
-    if (fqdnLocIpDoc != null){
-      updateProtocolDocument(fqdnLocIpDoc)
-    }
-
-    if (fqdnLocIpRow != null){
-      val fqdn = fqdnLocIpRow.getAs[String]("FQDN")
-      val serverIp = fqdnLocIpRow.getAs[String]("common_server_ip")
-      val firstFoundTime = fqdnLocIpRow.getAs[Long]("FIRST_FOUND_TIME")
-      val lastFoundTime = fqdnLocIpRow.getAs[Long]("LAST_FOUND_TIME")
-      val countTotalList = fqdnLocIpRow.getAs[ofRef[AnyRef]]("COUNT_TOTAL_LIST")
-      val schemaTypeList = fqdnLocIpRow.getAs[ofRef[AnyRef]]("schema_type_list")
-      val distCipRecent = fqdnLocIpRow.getAs[ofRef[String]]("DIST_CIP_RECENT")
-
-      val sepAttritubeMap: Map[String, Long] = separateAttributeByProtocol(schemaTypeList, countTotalList)
-      val distinctIp: Array[String] = mergeDistinctIp(distCipRecent)
-
-      val key = fqdn.concat("-" + serverIp)
-
-      if (fqdnLocIpDoc != null) {
-        updateMaxAttribute(fqdnLocIpDoc, lastFoundTime, "LAST_FOUND_TIME")
-        updateProtocolAttritube(fqdnLocIpDoc, sepAttritubeMap)
-        updateDistinctIp(fqdnLocIpDoc, distinctIp)
-      } else {
-        fqdnLocIpDoc = new BaseEdgeDocument()
-        fqdnLocIpDoc.setKey(key)
-        fqdnLocIpDoc.setFrom("FQDN/" + fqdn)
-        fqdnLocIpDoc.setTo("IP/" + serverIp)
-        fqdnLocIpDoc.addAttribute("FIRST_FOUND_TIME", firstFoundTime)
-        fqdnLocIpDoc.addAttribute("LAST_FOUND_TIME", lastFoundTime)
-        putProtocolAttritube(fqdnLocIpDoc, sepAttritubeMap)
-        putDistinctIp(fqdnLocIpDoc, distinctIp)
-      }
-    }
-
-    fqdnLocIpDoc
   }
 
 }

ip-learning-spark/src/main/scala/cn/ac/iie/spark/ArangoSpark.scala

@@ -1,136 +0,0 @@
-/*
- * DISCLAIMER
- *
- * Copyright 2016 ArangoDB GmbH, Cologne, Germany
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Copyright holder is ArangoDB GmbH, Cologne, Germany
- *
- * author Mark - mark at arangodb.com
- */
-
-package cn.ac.iie.spark
-
-import cn.ac.iie.spark.rdd.{ArangoRdd, ReadOptions, WriteOptions}
-import cn.ac.iie.spark.vpack.VPackUtils
-import com.arangodb.model.DocumentCreateOptions
-import org.apache.spark.SparkContext
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, Dataset, Row}
-
-import scala.collection.JavaConverters.seqAsJavaListConverter
-import scala.reflect.ClassTag
-
-object ArangoSpark {
-
-  /**
-   * Save data from rdd into ArangoDB
-   *
-   * @param rdd the rdd with the data to save
-   * @param collection the collection to save in
-   */
-  def save[T](rdd: RDD[T], collection: String): Unit =
-    save(rdd, collection, WriteOptions())
-
-  /**
-   * Save data from rdd into ArangoDB
-   *
-   * @param rdd the rdd with the data to save
-   * @param collection the collection to save in
-   * @param options additional write options
-   */
-  def save[T](rdd: RDD[T], collection: String, options: WriteOptions): Unit =
-    saveRDD(rdd, collection, options, (x: Iterator[T]) => x)
-
-  /**
-   * Save data from dataset into ArangoDB
-   *
-   * @param dataset the dataset with data to save
-   * @param collection the collection to save in
-   */
-  def save[T](dataset: Dataset[T], collection: String): Unit =
-    saveRDD(dataset.rdd, collection, WriteOptions(), (x: Iterator[T]) => x)
-
-  /**
-   * Save data from dataset into ArangoDB
-   *
-   * @param dataset the dataset with data to save
-   * @param collection the collection to save in
-   * @param options additional write options
-   */
-  def save[T](dataset: Dataset[T], collection: String, options: WriteOptions): Unit =
-    saveRDD(dataset.rdd, collection, options, (x: Iterator[T]) => x)
-
-  /**
-   * Save data from dataframe into ArangoDB
-   *
-   * @param dataframe the dataframe with data to save
-   * @param collection the collection to save in
-   */
-  def saveDF(dataframe: DataFrame, collection: String): Unit =
-    saveRDD[Row](dataframe.rdd, collection, WriteOptions(), (x: Iterator[Row]) => x.map { y => VPackUtils.rowToVPack(y) })
-
-  /**
-   * Save data from dataframe into ArangoDB
-   *
-   * @param dataframe the dataframe with data to save
-   * @param collection the collection to save in
-   * @param options additional write options
-   */
-  def saveDF(dataframe: DataFrame, collection: String, options: WriteOptions): Unit =
-    saveRDD[Row](dataframe.rdd, collection, options, (x: Iterator[Row]) => x.map { y => VPackUtils.rowToVPack(y) })
-
-  private def saveRDD[T](rdd: RDD[T], collection: String, options: WriteOptions, map: Iterator[T] => Iterator[Any]): Unit = {
-    val writeOptions = createWriteOptions(options, rdd.sparkContext.getConf)
-    rdd.foreachPartition { p =>
-      if (p.nonEmpty) {
-        val arangoDB = createArangoBuilder(writeOptions).build()
-        val col = arangoDB.db(writeOptions.database).collection(collection)
-        val docs = map(p).toList.asJava
-        writeOptions.method match {
-          case WriteOptions.INSERT  => col.insertDocuments(docs)
-          case WriteOptions.UPDATE  => col.updateDocuments(docs)
-          case WriteOptions.REPLACE => col.replaceDocuments(docs)
-          case WriteOptions.OVERWRITE =>
-            val documentCreateOptions = new DocumentCreateOptions
-            documentCreateOptions.overwrite(true)
-            documentCreateOptions.silent(true)
-            col.insertDocuments(docs, documentCreateOptions)
-        }
-
-        arangoDB.shutdown()
-      }
-    }
-  }
-
-  /**
-   * Load data from ArangoDB into rdd
-   *
-   * @param sparkContext the sparkContext containing the ArangoDB configuration
-   * @param collection the collection to load data from
-   */
-  def load[T: ClassTag](sparkContext: SparkContext, collection: String): ArangoRdd[T] =
-    load(sparkContext, collection, ReadOptions())
-
-  /**
-   * Load data from ArangoDB into rdd
-   *
-   * @param sparkContext the sparkContext containing the ArangoDB configuration
-   * @param collection the collection to load data from
-   * @param options read options
-   */
-  def load[T: ClassTag](sparkContext: SparkContext, collection: String, options: ReadOptions): ArangoRdd[T] =
-    new ArangoRdd[T](sparkContext, createReadOptions(options, sparkContext.getConf).copy(collection = collection))
-
-}

ip-learning-spark/src/main/scala/cn/ac/iie/spark/partition/QueryArangoPartition.scala

@@ -1,7 +0,0 @@
-package cn.ac.iie.spark.partition
-
-import org.apache.spark.Partition
-
-class QueryArangoPartition(idx: Int, val offset: Long, val separate: Long) extends Partition{
-  override def index: Int = idx
-}

ip-learning-spark/src/main/scala/cn/ac/iie/spark/rdd/ArangoOptions.scala

@@ -1,34 +0,0 @@
-package cn.ac.iie.spark.rdd
-
-import com.arangodb.Protocol
-import com.arangodb.entity.LoadBalancingStrategy
-
-trait ArangoOptions {
-
-  def database: String = "_system"
-
-  def hosts: Option[String] = None
-
-  def user: Option[String] = None
-
-  def password: Option[String] = None
-
-  def useSsl: Option[Boolean] = None
-
-  def sslKeyStoreFile: Option[String] = None
-
-  def sslPassPhrase: Option[String] = None
-
-  def sslProtocol: Option[String] = None
-
-  def protocol: Option[Protocol] = None
-  
-  def maxConnections: Option[Int] = None
-  
-  def acquireHostList: Option[Boolean] = None
-  
-  def acquireHostListInterval: Option[Int] = None
-  
-  def loadBalancingStrategy: Option[LoadBalancingStrategy] = None
-  
-}

ip-learning-spark/src/main/scala/cn/ac/iie/spark/rdd/ArangoRdd.scala

@@ -1,81 +0,0 @@
-package cn.ac.iie.spark.rdd
-
-import scala.collection.JavaConverters.asScalaIteratorConverter
-import cn.ac.iie.config.ApplicationConfig
-import cn.ac.iie.service.update.UpdateDocument
-import cn.ac.iie.spark
-import cn.ac.iie.spark.partition.QueryArangoPartition
-import com.arangodb.ArangoCursor
-import org.apache.spark.{Partition, SparkContext, TaskContext}
-import org.apache.spark.rdd.RDD
-import org.slf4j.LoggerFactory
-
-import scala.collection.mutable.ArrayBuffer
-import scala.reflect.ClassTag
-
-class ArangoRdd[T: ClassTag](@transient override val sparkContext: SparkContext,
-                             val options: ReadOptions
-                            ) extends RDD[T](sparkContext, Nil) {
-
-  private val LOG = LoggerFactory.getLogger(UpdateDocument.getClass)
-
-  override def compute(split: Partition, context: TaskContext): Iterator[T] = {
-
-    createCursor(split.asInstanceOf[QueryArangoPartition]).asScala
-  }
-
-  override protected def getPartitions: Array[Partition] = {
-    val partitions = ArrayBuffer[Partition]()
-    val total = getCountTotal
-    for (i <- 0 until ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS) {
-      val partition = getPartition(i, total)
-      partitions += partition
-    }
-    partitions.toArray
-  }
-
-  private def createCursor(split: QueryArangoPartition)(implicit clazz: ClassTag[T]): ArangoCursor[T] = {
-
-    var arangoCursor:ArangoCursor[T] = null
-    val arangoDB = spark.createArangoBuilder(options).build()
-    try {
-      val offset = split.offset
-      val separate = split.separate
-      val collection = options.collection
-      val sql = s"FOR doc IN $collection limit $offset,$separate RETURN doc"
-      LOG.info(sql)
-      arangoCursor = arangoDB.db(options.database).query(sql,clazz.runtimeClass.asInstanceOf[Class[T]])
-    }catch {
-      case e: Exception => LOG.error(s"创建Cursor异常:${e.getMessage}")
-    }finally {
-      arangoDB.shutdown()
-    }
-    arangoCursor
-  }
-
-  override def repartition(numPartitions: Int)(implicit ord: Ordering[T]): RDD[T] = super.repartition(numPartitions)
-
-  private def getPartition(idx: Int, countTotal: Long): QueryArangoPartition = {
-    val sepNum = countTotal / ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS + 1
-    val offsetNum = idx * sepNum
-    new QueryArangoPartition(idx, offsetNum, sepNum)
-  }
-
-  override def count(): Long = getCountTotal
-
-  private def getCountTotal: Long = {
-    val arangoDB = spark.createArangoBuilder(options).build()
-    var cnt = 0L
-    val sql = s"RETURN LENGTH(${options.collection})"
-    LOG.info(sql)
-    try {
-      val longs = arangoDB.db(options.database).query(sql, classOf[Long])
-      while (longs.hasNext) cnt = longs.next
-    } catch {
-      case e: Exception => LOG.error(sql + s"执行异常:${e.getMessage}")
-    }finally {
-      arangoDB.shutdown()
-    }
-    cnt
-  }
-}

ip-learning-spark/src/main/scala/cn/ac/iie/spark/rdd/ReadOptions.scala

@@ -1,93 +0,0 @@
-/*
- * DISCLAIMER
- *
- * Copyright 2016 ArangoDB GmbH, Cologne, Germany
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Copyright holder is ArangoDB GmbH, Cologne, Germany
- *
- * author Mark - mark at arangodb.com
- */
-
-package cn.ac.iie.spark.rdd
-
-import cn.ac.iie.spark.partition.QueryArangoPartition
-import com.arangodb.Protocol
-import com.arangodb.entity.LoadBalancingStrategy
-
-case class ReadOptions(override val database: String = "_system",
-                       val collection: String = null,
-                       partitioner: QueryArangoPartition = new QueryArangoPartition(0,0,0),
-                       override val hosts: Option[String] = None,
-                       override val user: Option[String] = None,
-                       override val password: Option[String] = None,
-                       override val useSsl: Option[Boolean] = None,
-                       override val sslKeyStoreFile: Option[String] = None,
-                       override val sslPassPhrase: Option[String] = None,
-                       override val sslProtocol: Option[String] = None,
-                       override val protocol: Option[Protocol] = None,
-                       override val maxConnections: Option[Int] = None,
-                       override val acquireHostList: Option[Boolean] = None,
-                       override val acquireHostListInterval: Option[Int] = None,
-                       override val loadBalancingStrategy: Option[LoadBalancingStrategy] = None) extends ArangoOptions {
-
-  def this() = this(database = "_system")
-
-  def database(database: String): ReadOptions = copy(database = database)
-
-  def collection(collection: String): ReadOptions = copy(collection = collection)
-
-  def hosts(hosts: String): ReadOptions = copy(hosts = Some(hosts))
-
-  def user(user: String): ReadOptions = copy(user = Some(user))
-
-  def password(password: String): ReadOptions = copy(password = Some(password))
-
-  def useSsl(useSsl: Boolean): ReadOptions = copy(useSsl = Some(useSsl))
-
-  def sslKeyStoreFile(sslKeyStoreFile: String): ReadOptions = copy(sslKeyStoreFile = Some(sslKeyStoreFile))
-
-  def sslPassPhrase(sslPassPhrase: String): ReadOptions = copy(sslPassPhrase = Some(sslPassPhrase))
-
-  def sslProtocol(sslProtocol: String): ReadOptions = copy(sslProtocol = Some(sslProtocol))
-
-  def protocol(protocol: Protocol): ReadOptions = copy(protocol = Some(protocol))
-  
-  def maxConnections(maxConnections: Int): ReadOptions = copy(maxConnections = Some(maxConnections))
-  
-  def acquireHostList(acquireHostList: Boolean): ReadOptions = copy(acquireHostList = Some(acquireHostList))
-  
-  def acquireHostListInterval(acquireHostListInterval: Int): ReadOptions = copy(acquireHostListInterval = Some(acquireHostListInterval))
-  
-  def loadBalancingStrategy(loadBalancingStrategy: LoadBalancingStrategy): ReadOptions = copy(loadBalancingStrategy = Some(loadBalancingStrategy))
-
-  def copy(database: String = database,
-           collection: String = collection,
-           partitioner: QueryArangoPartition = partitioner,
-           hosts: Option[String] = hosts,
-           user: Option[String] = user,
-           password: Option[String] = password,
-           useSsl: Option[Boolean] = useSsl,
-           sslKeyStoreFile: Option[String] = sslKeyStoreFile,
-           sslPassPhrase: Option[String] = sslPassPhrase,
-           sslProtocol: Option[String] = sslProtocol,
-           protocol: Option[Protocol] = protocol,
-           maxConnections: Option[Int] = maxConnections,
-           acquireHostList: Option[Boolean] = acquireHostList,
-           acquireHostListInterval: Option[Int] = acquireHostListInterval,
-           loadBalancingStrategy: Option[LoadBalancingStrategy] = loadBalancingStrategy): ReadOptions = {
-    ReadOptions(database, collection, partitioner, hosts, user, password, useSsl, sslKeyStoreFile, sslPassPhrase, sslProtocol, protocol, maxConnections, acquireHostList, acquireHostListInterval, loadBalancingStrategy)
-  }
-
-}

ip-learning-spark/src/main/scala/cn/ac/iie/spark/rdd/WriteOptions.scala

@@ -1,124 +0,0 @@
-/*
- * DISCLAIMER
- *
- * Copyright 2016 ArangoDB GmbH, Cologne, Germany
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Copyright holder is ArangoDB GmbH, Cologne, Germany
- *
- * author Mark - mark at arangodb.com
- */
-
-package cn.ac.iie.spark.rdd
-
-import javax.net.ssl.SSLContext
-import com.arangodb.Protocol
-import com.arangodb.entity.LoadBalancingStrategy
-
-case class WriteOptions(override val database: String = "_system",
-                        val method: WriteOptions.Method = WriteOptions.INSERT,
-                        override val hosts: Option[String] = None,
-                        override val user: Option[String] = None,
-                        override val password: Option[String] = None,
-                        override val useSsl: Option[Boolean] = None,
-                        override val sslKeyStoreFile: Option[String] = None,
-                        override val sslPassPhrase: Option[String] = None,
-                        override val sslProtocol: Option[String] = None,
-                        override val protocol: Option[Protocol] = None,
-                        override val maxConnections: Option[Int] = None,
-                        override val acquireHostList: Option[Boolean] = None,
-                        override val acquireHostListInterval: Option[Int] = None,
-                        override val loadBalancingStrategy: Option[LoadBalancingStrategy] = None) extends ArangoOptions {
-  import WriteOptions._
-
-  def this() = this(database = "_system")
-
-  def database(database: String): WriteOptions = copy(database = database)
-
-  def method(method: Method): WriteOptions = copy(method = method)
-
-  def hosts(hosts: String): WriteOptions = copy(hosts = Some(hosts))
-
-  def user(user: String): WriteOptions = copy(user = Some(user))
-
-  def password(password: String): WriteOptions = copy(password = Some(password))
-
-  def useSsl(useSsl: Boolean): WriteOptions = copy(useSsl = Some(useSsl))
-
-  def sslKeyStoreFile(sslKeyStoreFile: String): WriteOptions = copy(sslKeyStoreFile = Some(sslKeyStoreFile))
-
-  def sslPassPhrase(sslPassPhrase: String): WriteOptions = copy(sslPassPhrase = Some(sslPassPhrase))
-
-  def sslProtocol(sslProtocol: String): WriteOptions = copy(sslProtocol = Some(sslProtocol))
-
-  def protocol(protocol: Protocol): WriteOptions = copy(protocol = Some(protocol))
-
-  def maxConnections(maxConnections: Int): WriteOptions = copy(maxConnections = Some(maxConnections))
-
-  def acquireHostList(acquireHostList: Boolean): WriteOptions = copy(acquireHostList = Some(acquireHostList))
-
-  def acquireHostListInterval(acquireHostListInterval: Int): WriteOptions = copy(acquireHostListInterval = Some(acquireHostListInterval))
-
-  def loadBalancingStrategy(loadBalancingStrategy: LoadBalancingStrategy): WriteOptions = copy(loadBalancingStrategy = Some(loadBalancingStrategy))
-
-  def copy(database: String = database,
-           method: Method = method,
-           hosts: Option[String] = hosts,
-           user: Option[String] = user,
-           password: Option[String] = password,
-           useSsl: Option[Boolean] = useSsl,
-           sslKeyStoreFile: Option[String] = sslKeyStoreFile,
-           sslPassPhrase: Option[String] = sslPassPhrase,
-           sslProtocol: Option[String] = sslProtocol,
-           protocol: Option[Protocol] = protocol,
-           maxConnections: Option[Int] = maxConnections,
-           acquireHostList: Option[Boolean] = acquireHostList,
-           acquireHostListInterval: Option[Int] = acquireHostListInterval,
-           loadBalancingStrategy: Option[LoadBalancingStrategy] = loadBalancingStrategy): WriteOptions = {
-    WriteOptions(database, method, hosts, user, password, useSsl, sslKeyStoreFile, sslPassPhrase, sslProtocol, protocol, maxConnections, acquireHostList, acquireHostListInterval, loadBalancingStrategy)
-  }
-
-}
-
-object WriteOptions {
-
-  /**
-    * method to save documents to arangodb
-    */
-  sealed trait Method
-
-  /**
-    * save documents by inserting
-    * @see [[com.arangodb.ArangoCollection#insertDocuments(java.util.Collection)]]
-    */
-  case object INSERT extends Method
-
-  /**
-    * save documents by updating
-    * @see [[com.arangodb.ArangoCollection#updateDocuments(java.util.Collection)]]
-    */
-  case object UPDATE extends Method
-
-  /**
-    * save documents by replacing
-    * @see [[com.arangodb.ArangoCollection#replaceDocuments(java.util.Collection)]]
-    */
-  case object REPLACE extends Method
-
-  /**
-    * save documents by overwrite
-    */
-  case object OVERWRITE extends Method
-
-}

ip-learning-spark/src/main/scala/cn/ac/iie/spark/spark.scala

@@ -1,139 +0,0 @@
-/*
- * DISCLAIMER
- *
- * Copyright 2016 ArangoDB GmbH, Cologne, Germany
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Copyright holder is ArangoDB GmbH, Cologne, Germany
- *
- * author Mark - mark at arangodb.com
- */
-
-package cn.ac.iie
-
-import java.io.FileInputStream
-import java.security.KeyStore
-
-import cn.ac.iie.spark.rdd.{ArangoOptions, ReadOptions, WriteOptions}
-import com.arangodb.{ArangoDB, ArangoDBException, Protocol}
-import com.arangodb.entity.LoadBalancingStrategy
-import com.arangodb.velocypack.module.jdk8.VPackJdk8Module
-import com.arangodb.velocypack.module.scala.VPackScalaModule
-import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory}
-import org.apache.spark.SparkConf
-
-import scala.util.Try
-
-package object spark {
-
-  val PropertyHosts = "arangodb.hosts"
-  val PropertyUser = "arangodb.user"
-  val PropertyPassword = "arangodb.password"
-  val PropertyUseSsl = "arangodb.useSsl"
-  val PropertySslKeyStoreFile = "arangodb.ssl.keyStoreFile"
-  val PropertySslPassPhrase = "arangodb.ssl.passPhrase"
-  val PropertySslProtocol = "arangodb.ssl.protocol"
-  val PropertyProtocol = "arangodb.protocol"
-  val PropertyMaxConnections = "arangodb.maxConnections"
-  val PropertyAcquireHostList = "arangodb.acquireHostList"
-  val PropertyAcquireHostListInterval = "arangodb.acquireHostListInterval"
-  val PropertyLoadBalancingStrategy = "arangodb.loadBalancingStrategy"
-
-  private[spark] def createReadOptions(options: ReadOptions, sc: SparkConf): ReadOptions = {
-    options.copy(
-      hosts = options.hosts.orElse(some(sc.get(PropertyHosts, null))),
-      user = options.user.orElse(some(sc.get(PropertyUser, null))),
-      password = options.password.orElse(some(sc.get(PropertyPassword, null))),
-      useSsl = options.useSsl.orElse(some(Try(sc.get(PropertyUseSsl, null).toBoolean).getOrElse(false))),
-      sslKeyStoreFile = options.sslKeyStoreFile.orElse(some(sc.get(PropertySslKeyStoreFile, null))),
-      sslPassPhrase = options.sslPassPhrase.orElse(some(sc.get(PropertySslPassPhrase, null))),
-      sslProtocol = options.sslProtocol.orElse(some(sc.get(PropertySslProtocol, null))),
-      protocol = options.protocol.orElse(some(Protocol.valueOf(sc.get(PropertyProtocol, "VST")))),
-      maxConnections = options.maxConnections.orElse(some(Try(sc.get(PropertyMaxConnections, null).toInt).getOrElse(1))),
-      acquireHostList = options.acquireHostList.orElse(some(Try(sc.get(PropertyAcquireHostList, null).toBoolean).getOrElse(false))),
-      acquireHostListInterval = options.acquireHostListInterval.orElse(some(Try(sc.get(PropertyAcquireHostListInterval, null).toInt).getOrElse(60000))),
-      loadBalancingStrategy = options.loadBalancingStrategy.orElse(some(LoadBalancingStrategy.valueOf(sc.get(PropertyLoadBalancingStrategy, "NONE")))))
-  }
-
-  private[spark] def createWriteOptions(options: WriteOptions, sc: SparkConf): WriteOptions = {
-    options.copy(
-      hosts = options.hosts.orElse(some(sc.get(PropertyHosts, null))),
-      user = options.user.orElse(some(sc.get(PropertyUser, null))),
-      password = options.password.orElse(some(sc.get(PropertyPassword, null))),
-      useSsl = options.useSsl.orElse(some(Try(sc.get(PropertyUseSsl, null).toBoolean).getOrElse(false))),
-      sslKeyStoreFile = options.sslKeyStoreFile.orElse(some(sc.get(PropertySslKeyStoreFile, null))),
-      sslPassPhrase = options.sslPassPhrase.orElse(some(sc.get(PropertySslPassPhrase, null))),
-      sslProtocol = options.sslProtocol.orElse(some(sc.get(PropertySslProtocol, null))),
-      protocol = options.protocol.orElse(some(Protocol.valueOf(sc.get(PropertyProtocol, "VST")))),
-      maxConnections = options.maxConnections.orElse(some(Try(sc.get(PropertyMaxConnections, null).toInt).getOrElse(1))),
-      acquireHostList = options.acquireHostList.orElse(some(Try(sc.get(PropertyAcquireHostList, null).toBoolean).getOrElse(false))),
-      acquireHostListInterval = options.acquireHostListInterval.orElse(some(Try(sc.get(PropertyAcquireHostListInterval, null).toInt).getOrElse(60000))),
-      loadBalancingStrategy = options.loadBalancingStrategy.orElse(some(LoadBalancingStrategy.valueOf(sc.get(PropertyLoadBalancingStrategy, "NONE")))))
-  }
-
-  private[spark] def createArangoBuilder(options: ArangoOptions): ArangoDB.Builder = {
-    val builder = new ArangoDB.Builder()
-    builder.registerModules(new VPackJdk8Module, new VPackScalaModule)
-    options.hosts.foreach { hosts(_).foreach(host => builder.host(host._1, host._2)) }
-    options.user.foreach { builder.user(_) }
-    options.password.foreach { builder.password(_) }
-    options.useSsl.foreach { builder.useSsl(_) }
-    if (options.sslKeyStoreFile.isDefined && options.sslPassPhrase.isDefined) {
-      builder.sslContext(createSslContext(options.sslKeyStoreFile.get, options.sslPassPhrase.get, options.sslProtocol.getOrElse("TLS")))
-    }
-    options.protocol.foreach { builder.useProtocol(_) }
-    options.maxConnections.foreach { builder.maxConnections(_) }
-    options.acquireHostList.foreach { builder.acquireHostList(_) }
-    options.acquireHostListInterval.foreach { builder.acquireHostListInterval(_) }
-    options.loadBalancingStrategy.foreach { builder.loadBalancingStrategy(_) }
-    builder
-  }
-
-  private def createSslContext(keyStoreFile: String, passPhrase: String, protocol: String): SSLContext = {
-    val ks = KeyStore.getInstance(KeyStore.getDefaultType());
-    val kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-    ks.load(new FileInputStream(keyStoreFile), passPhrase.toCharArray());
-    kmf.init(ks, passPhrase.toCharArray());
-    val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-    tmf.init(ks);
-    val sc = SSLContext.getInstance(protocol);
-    sc.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
-    sc
-  }
-
-  private def some(value: String): Option[String] =
-    if (value != null) Some(value) else None
-
-  private def some(value: Int): Option[Int] =
-    Some(value)
-    
-  private def some(value: Boolean): Option[Boolean] =
-    Some(value)
-
-  private def some(value: Protocol): Option[Protocol] =
-    Some(value)
-    
-  private def some(value: LoadBalancingStrategy): Option[LoadBalancingStrategy] =
-    Some(value)
-
-  private def hosts(hosts: String): List[(String, Int)] =
-    hosts.split(",").map({ x =>
-      val s = x.split(":")
-      if (s.length != 2 || !s(1).matches("[0-9]+"))
-        throw new ArangoDBException(s"Could not load property-value arangodb.hosts=${s}. Expected format ip:port,ip:port,...");
-      else
-        (s(0), s(1).toInt)
-    }).toList
-
-}

ip-learning-spark/src/main/scala/cn/ac/iie/spark/vpack/VPackUtils.scala

@@ -1,160 +0,0 @@
-/*
- * DISCLAIMER
- *
- * Copyright 2016 ArangoDB GmbH, Cologne, Germany
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Copyright holder is ArangoDB GmbH, Cologne, Germany
- *
- * author Mark - mark at arangodb.com
- */
-
-package cn.ac.iie.spark.vpack
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.types.{
-  ArrayType,
-  BooleanType,
-  DataType,
-  DateType,
-  DecimalType,
-  DoubleType,
-  FloatType,
-  IntegerType,
-  LongType,
-  MapType,
-  NullType,
-  ShortType,
-  StringType,
-  StructField,
-  StructType,
-  TimestampType
-}
-import com.arangodb.velocypack.VPackBuilder
-import com.arangodb.velocypack.VPackSlice
-import com.arangodb.velocypack.ValueType
-
-private[spark] object VPackUtils {
-
-  def rowToVPack(row: Row): VPackSlice = {
-    val builder = new VPackBuilder()
-    if (row == null) {
-      builder.add(ValueType.NULL)
-    } else {
-      builder.add(ValueType.OBJECT)
-      row.schema.fields.zipWithIndex.foreach { addField(_, row, builder) }
-      builder.close()
-    }
-    builder.slice()
-  }
-
-  private def addField(field: (StructField, Int), row: Row, builder: VPackBuilder): Unit = {
-    val name = field._1.name
-    val index = field._2
-    if (row.isNullAt(index)) {
-      builder.add(name, ValueType.NULL)
-    } else {
-      field._1.dataType match {
-        case BooleanType    => builder.add(name, java.lang.Boolean.valueOf(row.getBoolean(index)))
-        case DoubleType     => builder.add(name, java.lang.Double.valueOf(row.getDouble(index)))
-        case FloatType      => builder.add(name, java.lang.Float.valueOf(row.getFloat(index)))
-        case LongType       => builder.add(name, java.lang.Long.valueOf(row.getLong(index)))
-        case IntegerType    => builder.add(name, java.lang.Integer.valueOf(row.getInt(index)))
-        case ShortType      => builder.add(name, java.lang.Short.valueOf(row.getShort(index)))
-        case StringType     => builder.add(name, java.lang.String.valueOf(row.getString(index)));
-        case DateType       => builder.add(name, row.getDate(index))
-        case TimestampType  => builder.add(name, row.getTimestamp(index))
-        case t: DecimalType => builder.add(name, row.getDecimal(index))
-        case t: MapType => {
-          builder.add(name, ValueType.OBJECT)
-          row.getMap[String, Any](index).foreach { case (name, value) => addValue(name, value, builder) }
-          builder.close()
-        }
-        case t: ArrayType => {
-          builder.add(name, ValueType.ARRAY)
-          addValues(row, index, builder, t.elementType)
-          builder.close()
-        }
-        case NullType           => builder.add(name, ValueType.NULL)
-        case struct: StructType => builder.add(name, rowToVPack(row.getStruct(index)))
-        case _                  => // TODO
-      }
-    }
-  }
-
-  private def addValues(row: Row, index: Int, builder: VPackBuilder, itemType: DataType): Unit = {
-    itemType match {
-      case BooleanType =>
-        row.getSeq[Boolean](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case DoubleType =>
-        row.getSeq[Double](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case FloatType =>
-        row.getSeq[Float](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case LongType =>
-        row.getSeq[Long](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case IntegerType =>
-        row.getSeq[Int](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case ShortType =>
-        row.getSeq[Short](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case StringType =>
-        row.getSeq[String](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case DateType =>
-        row.getSeq[java.sql.Date](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case TimestampType =>
-        row.getSeq[java.sql.Timestamp](index).foreach { value =>
-          addValue(null, value, builder)
-        }
-      case s: StructType => {
-        row.getSeq[Row](index).foreach { value =>
-          builder.add(null, rowToVPack(value))
-        }
-      }
-      case t: MapType   => // TODO
-      case t: ArrayType => // TODO
-      case _            => // TODO
-    }
-  }
-
-  private def addValue(name: String, value: Any, builder: VPackBuilder): Unit = {
-    value match {
-      case value: Boolean            => builder.add(name, java.lang.Boolean.valueOf(value))
-      case value: Double             => builder.add(name, java.lang.Double.valueOf(value))
-      case value: Float              => builder.add(name, java.lang.Float.valueOf(value))
-      case value: Long               => builder.add(name, java.lang.Long.valueOf(value))
-      case value: Int                => builder.add(name, java.lang.Integer.valueOf(value))
-      case value: Short              => builder.add(name, java.lang.Short.valueOf(value))
-      case value: String             => builder.add(name, java.lang.String.valueOf(value))
-      case value: java.sql.Date      => builder.add(name, value)
-      case value: java.sql.Timestamp => builder.add(name, value)
-      case _                         => // TODO
-    }
-  }
-
-}

ip-learning-spark/src/main/scala/cn/ac/iie/utils/SparkSessionUtil.scala

@@ -1,9 +1,7 @@
 package cn.ac.iie.utils
 
 import cn.ac.iie.config.ApplicationConfig
-import org.apache.spark.SparkContext
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.util.LongAccumulator
 import org.slf4j.LoggerFactory
 
 object SparkSessionUtil {
@@ -11,8 +9,6 @@ object SparkSessionUtil {
 
   val spark: SparkSession = getSparkSession
 
-  var sparkContext: SparkContext = getContext
-
   private def getSparkSession: SparkSession ={
     val spark: SparkSession = SparkSession
       .builder()
@@ -21,36 +17,16 @@ object SparkSessionUtil {
       .config("spark.network.timeout", ApplicationConfig.SPARK_NETWORK_TIMEOUT)
       .config("spark.sql.shuffle.partitions", ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS)
       .config("spark.executor.memory", ApplicationConfig.SPARK_EXECUTOR_MEMORY)
-      .config("arangodb.hosts", s"${ApplicationConfig.ARANGODB_HOST}:${ApplicationConfig.ARANGODB_PORT}")
-      .config("arangodb.user", ApplicationConfig.ARANGODB_USER)
-      .config("arangodb.password", ApplicationConfig.ARANGODB_PASSWORD)
       .master(ApplicationConfig.MASTER)
       .getOrCreate()
     LOG.warn("sparkession获取成功！！！")
     spark
   }
 
-  def getContext: SparkContext = {
-    @transient var sc: SparkContext = null
-      if (sparkContext == null) sc = spark.sparkContext
-    sc
-  }
-
-  def getLongAccumulator(name: String): LongAccumulator ={
-    if (sparkContext == null){
-      sparkContext = getContext
-    }
-    sparkContext.longAccumulator(name)
-
-  }
-
   def closeSpark(): Unit ={
     if (spark != null){
       spark.stop()
     }
-    if (sparkContext != null){
-      sparkContext.stop()
-    }
   }
 
 }

ip-learning-spark/src/test/scala/cn/ac/iie/dao/BaseClickhouseDataTest.scala

@@ -7,7 +7,7 @@ import org.apache.spark.sql.SparkSession
 object BaseClickhouseDataTest {
   private val spark: SparkSession = SparkSessionUtil.spark
   def main(args: Array[String]): Unit = {
-//    BaseClickhouseData loadConnectionDataFromCk()
+    BaseClickhouseData loadConnectionDataFromCk()
     val sql =
       """
         |SELECT

ip-learning-spark/src/test/scala/cn/ac/iie/service/update/UpdateDocumentTest.scala

@@ -0,0 +1,35 @@
+//package cn.ac.iie.service.update
+//
+//import java.util
+//import java.util.ArrayList
+//import java.util.concurrent.ConcurrentHashMap
+//
+//import cn.ac.iie.dao.BaseArangoData
+//import cn.ac.iie.dao.BaseArangoData._
+//import com.arangodb.entity.{BaseDocument, BaseEdgeDocument}
+//
+//import scala.collection.mutable.WrappedArray.ofRef
+//
+//object UpdateDocumentTest {
+//  def main(args: Array[String]): Unit = {
+//    val baseArangoData = new BaseArangoData()
+//    baseArangoData.readHistoryData("R_LOCATE_FQDN2IP", historyRelationFqdnAddressIpMap, classOf[BaseEdgeDocument])
+//
+//    val value = BaseArangoData.historyRelationFqdnAddressIpMap.keys()
+//    while (value.hasMoreElements) {
+//      val integer: Integer = value.nextElement()
+//      val map: ConcurrentHashMap[String, BaseEdgeDocument] = historyRelationFqdnAddressIpMap.get(integer)
+//      val unit = map.keys()
+//      while (unit.hasMoreElements) {
+//        val key = unit.nextElement()
+//        val edgeDocument = map.get(key)
+//        //        val longs = edgeDocument.getAttribute("DNS_CNT_RECENT").asInstanceOf[util.ArrayList[Long]]
+//        //        val strings = edgeDocument.getAttribute("DIST_CIP").asInstanceOf[util.ArrayList[String]]
+//        val strings = edgeDocument.getAttribute("DIST_CIP").asInstanceOf[Array[String]]
+//        val longs = edgeDocument.getAttribute("DNS_CNT_RECENT").asInstanceOf[Array[java.lang.Long]]
+//        println(longs.toString + "---" + strings.toString)
+//      }
+//    }
+//  }
+//
+//}

ip-learning-spark/src/test/scala/cn/ac/iie/spark/RDDTest.scala

@@ -1,75 +0,0 @@
-package cn.ac.iie.spark
-
-import cn.ac.iie.config.ApplicationConfig
-import cn.ac.iie.dao.BaseClickhouseData
-import cn.ac.iie.spark.partition.CustomPartitioner
-import cn.ac.iie.spark.rdd.{ArangoRdd, ReadOptions}
-import cn.ac.iie.utils.SparkSessionUtil
-import com.arangodb.entity.BaseDocument
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.functions.{collect_list, max, min}
-import org.apache.spark.storage.StorageLevel
-
-object RDDTest {
-  def main(args: Array[String]): Unit = {
-
-    val sparkContext = SparkSessionUtil.spark.sparkContext
-
-    println(sparkContext.getConf.get("arangodb.hosts"))
-
-    //    val options = ReadOptions("iplearn_media_domain").copy(collection = "R_LOCATE_FQDN2IP")
-    val options = ReadOptions(ApplicationConfig.ARANGODB_DB_NAME)
-
-    val ipOptions = options.copy(collection = "IP")
-
-    val rdd: ArangoRdd[BaseDocument] = ArangoSpark.load[BaseDocument](sparkContext,"IP",options)
-
-    println(rdd.count())
-    println(rdd.getNumPartitions)
-
-    val ipRDD = mergeVertexIp()
-    val value: RDD[(String, (Option[BaseDocument], Option[Row]))] = rdd.map(doc => {
-      (doc.getKey, doc)
-    }).fullOuterJoin(ipRDD)
-    value.foreach((row: (String, (Option[BaseDocument], Option[Row]))) => {
-      val value = row._2._2
-      val str: String = value match {
-        case Some(r) => r.getAs[String]("IP")
-//        case None => null
-        case _ => null
-      }
-      println(str)
-    })
-
-    /*
-    val value: RDD[BaseDocument] = rdd.filter(doc => doc.getAttribute("CLIENT_SESSION_COUNT").asInstanceOf[Long] > 100).map(doc => {
-      doc.addAttribute("abc", 1)
-      doc
-    })
-    value.map(doc => {(doc.getKey,doc)})
-    value.persist(StorageLevel.MEMORY_AND_DISK)
-    value.foreach(fqdnRow => println(fqdnRow.toString))
-    println(value.count())
-    */
-
-    SparkSessionUtil.spark.close()
-    System.exit(0)
-
-  }
-
-  def mergeVertexIp(): RDD[(String,Row)]={
-    val vertexIpDf = BaseClickhouseData.getVertexIpDf
-    val frame = vertexIpDf.groupBy("IP").agg(
-      min("FIRST_FOUND_TIME").alias("FIRST_FOUND_TIME"),
-      max("LAST_FOUND_TIME").alias("LAST_FOUND_TIME"),
-      collect_list("SESSION_COUNT").alias("SESSION_COUNT_LIST"),
-      collect_list("BYTES_SUM").alias("BYTES_SUM_LIST"),
-      collect_list("ip_type").alias("ip_type_list")
-    )
-    val values = frame.rdd.map(row => (row.getAs[String]("IP"), row))
-      .partitionBy(new CustomPartitioner(ApplicationConfig.SPARK_SQL_SHUFFLE_PARTITIONS))
-    values
-  }
-
-}