YSP分析spark测试

ip-learning-spark/src/main/java/cn/ac/iie/service/read/ReadHistoryArangoData.java

@@ -64,6 +64,13 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
                 int i = 0;
                 for (T doc : baseDocuments) {
                     String key = doc.getKey();
+                    switch (table) {
+                        case "R_LOCATE_FQDN2IP":
+//                            updateProtocolDocument(doc);
+                            deleteDistinctClientIpByTime(doc);
+                            break;
+                        default:
+                    }
                     int hashCode = Math.abs(key.hashCode()) % ApplicationConfig.THREAD_POOL_NUMBER();
                     ConcurrentHashMap<String, T> tmpMap = map.get(hashCode);
                     tmpMap.put(key, doc);

ip-learning-spark/src/main/resources/application.properties

@@ -29,7 +29,7 @@ thread.pool.number=5
 
 #读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围
 clickhouse.time.limit.type=1
-read.clickhouse.max.time=1571241720
+read.clickhouse.max.time=1571241640
 read.clickhouse.min.time=1571241600
 
 #读取arangoDB时间范围方式，0：正常读；1：指定时间范围