From cbeba6372b5964134bdc558e352394c8338cc9de Mon Sep 17 00:00:00 2001 From: wanglihui <949764788@qq.com> Date: Wed, 19 Aug 2020 10:48:08 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A0=B9=E6=8D=AEserver=E4=B8=8Eclient=20IP?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E5=8C=BA=E5=88=86=E9=93=BE=E8=B7=AF=E4=BF=A1?= =?UTF-8?q?=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ac/iie/service/ingestion/ReadClickhouseData.java | 10 ++++++---- .../iie/service/ingestion/ReadHistoryArangoData.java | 6 +++++- .../src/main/resources/application.properties | 3 ++- .../src/main/resources/clickhouse.properties | 8 +++++--- .../src/test/java/cn/ac/iie/readHistoryDataTest.java | 1 + 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java index 86981fd..de3735b 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java @@ -66,9 +66,11 @@ public class ReadClickhouseData { long bytesSum = resultSet.getLong("BYTES_SUM"); String ipType = resultSet.getString("ip_type"); String[] commonLinkInfos = (String[]) resultSet.getArray("common_link_info").getArray(); - String commonLinkInfo = ""; - if (commonLinkInfos.length > 1){ + String commonLinkInfo; + if (commonLinkInfos.length > 1 && !commonLinkInfos[1].equals("")){ commonLinkInfo = commonLinkInfos[1]; + }else { + commonLinkInfo = commonLinkInfos[0]; } newDoc.setKey(ip); newDoc.addAttribute("IP", ip); @@ -264,8 +266,8 @@ public class ReadClickhouseData { public static String getVertexIpSql() { String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime; - String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP"; - String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP"; + String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_c2s) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP"; + String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_s2c) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP"; return "SELECT * FROM((" + clientIpSql + ") UNION ALL (" + serverIpSql + "))"; } diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java index 5214fc4..d84ac45 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java @@ -8,6 +8,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -73,7 +74,7 @@ public class ReadHistoryArangoData extends Thread { LOG.info(query + "\n读取" + i + "条数据，运行时间：" + (l - s)); } } catch (Exception e) { - e.printStackTrace(); + LOG.error(Arrays.toString(e.getStackTrace())); } finally { countDownLatch.countDown(); LOG.info("本线程读取完毕，剩余线程数量：" + countDownLatch.getCount()); @@ -97,6 +98,9 @@ public class ReadHistoryArangoData extends Thread { private void deleteDistinctClientIpByTime(T doc) { ArrayList distCip = (ArrayList) doc.getAttribute("DIST_CIP"); ArrayList distCipTs = (ArrayList) doc.getAttribute("DIST_CIP_TS"); + if (distCip == null || distCip.isEmpty()){ + return; + } distCipTs.add(currentHour - RECENT_COUNT_HOUR * 3600); Collections.sort(distCipTs); Collections.reverse(distCipTs); diff --git a/IP-learning-graph/src/main/resources/application.properties b/IP-learning-graph/src/main/resources/application.properties index 7293dc1..cce0baf 100644 --- a/IP-learning-graph/src/main/resources/application.properties +++ b/IP-learning-graph/src/main/resources/application.properties @@ -4,7 +4,8 @@ arangoDB.host=192.168.40.182 arangoDB.port=8529 arangoDB.user=upsert arangoDB.password=ceiec2018 -arangoDB.DB.name=ip-learning-test +#arangoDB.DB.name=ip-learning-test +arangoDB.DB.name=ip-learning-test-0 #arangoDB.DB.name=tsg_galaxy_v3 arangoDB.batch=100000 arangoDB.ttl=3600 diff --git a/IP-learning-graph/src/main/resources/clickhouse.properties b/IP-learning-graph/src/main/resources/clickhouse.properties index 3b18aa4..339c5d5 100644 --- a/IP-learning-graph/src/main/resources/clickhouse.properties +++ b/IP-learning-graph/src/main/resources/clickhouse.properties @@ -1,9 +1,11 @@ drivers=ru.yandex.clickhouse.ClickHouseDriver mdb.user=default -db.id=192.168.40.186:8123/tsg_galaxy_v3?socket_timeout=300000 -mdb.password=111111 +#db.id=192.168.40.186:8123/tsg_galaxy_v3?socket_timeout=300000 +#mdb.password=111111 + +db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000&compress=0 #db.id=192.168.40.224:8123/tsg_galaxy_v3?socket_timeout=300000 -#mdb.password=ceiec2019 +mdb.password=ceiec2019 initialsize=1 minidle=1 maxactive=50 diff --git a/IP-learning-graph/src/test/java/cn/ac/iie/readHistoryDataTest.java b/IP-learning-graph/src/test/java/cn/ac/iie/readHistoryDataTest.java index 56f9b50..93c4e49 100644 --- a/IP-learning-graph/src/test/java/cn/ac/iie/readHistoryDataTest.java +++ b/IP-learning-graph/src/test/java/cn/ac/iie/readHistoryDataTest.java @@ -6,5 +6,6 @@ import com.arangodb.entity.BaseEdgeDocument; public class readHistoryDataTest { public static void main(String[] args) { BaseArangoData baseArangoData = new BaseArangoData(); + } }