From 611335c05375c341f2c9d0617d7a7e26d41fb630 Mon Sep 17 00:00:00 2001 From: wanglihui <949764788@qq.com> Date: Wed, 12 Aug 2020 15:53:55 +0800 Subject: [PATCH] =?UTF-8?q?YSP=E5=88=86=E6=9E=90spark=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/ac/iie/service/read/ReadHistoryArangoData.java | 7 +++++++ .../src/main/resources/application.properties | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ip-learning-spark/src/main/java/cn/ac/iie/service/read/ReadHistoryArangoData.java b/ip-learning-spark/src/main/java/cn/ac/iie/service/read/ReadHistoryArangoData.java index 69ceda8..26cb28c 100644 --- a/ip-learning-spark/src/main/java/cn/ac/iie/service/read/ReadHistoryArangoData.java +++ b/ip-learning-spark/src/main/java/cn/ac/iie/service/read/ReadHistoryArangoData.java @@ -64,6 +64,13 @@ public class ReadHistoryArangoData extends Thread { int i = 0; for (T doc : baseDocuments) { String key = doc.getKey(); + switch (table) { + case "R_LOCATE_FQDN2IP": +// updateProtocolDocument(doc); + deleteDistinctClientIpByTime(doc); + break; + default: + } int hashCode = Math.abs(key.hashCode()) % ApplicationConfig.THREAD_POOL_NUMBER(); ConcurrentHashMap tmpMap = map.get(hashCode); tmpMap.put(key, doc); diff --git a/ip-learning-spark/src/main/resources/application.properties b/ip-learning-spark/src/main/resources/application.properties index 8116da4..a8a01e5 100644 --- a/ip-learning-spark/src/main/resources/application.properties +++ b/ip-learning-spark/src/main/resources/application.properties @@ -29,7 +29,7 @@ thread.pool.number=5 #读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围 clickhouse.time.limit.type=1 -read.clickhouse.max.time=1571241720 +read.clickhouse.max.time=1571241640 read.clickhouse.min.time=1571241600 #读取arangoDB时间范围方式，0：正常读；1：指定时间范围