diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java index de3735b..da2d897 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java @@ -265,7 +265,7 @@ public class ReadClickhouseData { } public static String getVertexIpSql() { - String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime; + String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime + " AND common_schema_type != 'BASE'"; String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_c2s) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP"; String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_s2c) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP"; return "SELECT * FROM((" + clientIpSql + ") UNION ALL (" + serverIpSql + "))"; diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java index d84ac45..d8ded7e 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java @@ -99,6 +99,8 @@ public class ReadHistoryArangoData extends Thread { ArrayList distCip = (ArrayList) doc.getAttribute("DIST_CIP"); ArrayList distCipTs = (ArrayList) doc.getAttribute("DIST_CIP_TS"); if (distCip == null || distCip.isEmpty()){ + doc.updateAttribute("DIST_CIP", new String[0]); + doc.updateAttribute("DIST_CIP_TS", new long[0]); return; } distCipTs.add(currentHour - RECENT_COUNT_HOUR * 3600); diff --git a/IP-learning-graph/src/main/resources/application.properties b/IP-learning-graph/src/main/resources/application.properties index cce0baf..fec32ff 100644 --- a/IP-learning-graph/src/main/resources/application.properties +++ b/IP-learning-graph/src/main/resources/application.properties @@ -1,12 +1,9 @@ #arangoDB参数配置 -arangoDB.host=192.168.40.182 -#arangoDB.host=192.168.40.224 +arangoDB.host=192.168.44.12 arangoDB.port=8529 arangoDB.user=upsert arangoDB.password=ceiec2018 -#arangoDB.DB.name=ip-learning-test -arangoDB.DB.name=ip-learning-test-0 -#arangoDB.DB.name=tsg_galaxy_v3 +arangoDB.DB.name=tsg_galaxy_v3 arangoDB.batch=100000 arangoDB.ttl=3600 @@ -19,8 +16,8 @@ thread.await.termination.time=10 #读取clickhouse时间范围方式,0:读取过去一小时,1:指定时间范围 time.limit.type=0 -read.clickhouse.max.time=1596684142 -read.clickhouse.min.time=1596425769 +read.clickhouse.max.time=1598323368 +read.clickhouse.min.time=1597222501 update.interval=3600 distinct.client.ip.num=10000 diff --git a/IP-learning-graph/src/main/resources/clickhouse.properties b/IP-learning-graph/src/main/resources/clickhouse.properties index 339c5d5..4027174 100644 --- a/IP-learning-graph/src/main/resources/clickhouse.properties +++ b/IP-learning-graph/src/main/resources/clickhouse.properties @@ -1,10 +1,7 @@ drivers=ru.yandex.clickhouse.ClickHouseDriver mdb.user=default -#db.id=192.168.40.186:8123/tsg_galaxy_v3?socket_timeout=300000 -#mdb.password=111111 - -db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000&compress=0 -#db.id=192.168.40.224:8123/tsg_galaxy_v3?socket_timeout=300000 +#db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000 +db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0 mdb.password=ceiec2019 initialsize=1 minidle=1