diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java b/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java index b9ead95..03c65a2 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java @@ -47,11 +47,11 @@ public class UpdateGraphData { // updateDocument(newVertexFqdnMap, historyVertexFqdnMap, "FQDN", Fqdn.class,BaseDocument.class, // ReadClickhouseData::getVertexFqdnSql,ReadClickhouseData::getVertexFqdnDocument); -// updateDocument(newVertexIpMap,historyVertexIpMap,"IP", Ip.class,BaseDocument.class, -// ReadClickhouseData::getVertexIpSql,ReadClickhouseData::getVertexIpDocument); + updateDocument(newVertexIpMap,historyVertexIpMap,"IP", Ip.class,BaseDocument.class, + ReadClickhouseData::getVertexIpSql,ReadClickhouseData::getVertexIpDocument); -// updateDocument(newVertexSubscriberMap,historyVertexSubscriberMap,"SUBSCRIBER", Subscriber.class,BaseDocument.class, -// ReadClickhouseData::getVertexSubscriberSql,ReadClickhouseData::getVertexSubscriberDocument); + updateDocument(newVertexSubscriberMap,historyVertexSubscriberMap,"SUBSCRIBER", Subscriber.class,BaseDocument.class, + ReadClickhouseData::getVertexSubscriberSql,ReadClickhouseData::getVertexSubscriberDocument); updateDocument(newRelationFqdnAddressIpMap,historyRelationFqdnAddressIpMap,"R_LOCATE_FQDN2IP", LocateFqdn2Ip.class,BaseEdgeDocument.class, ReadClickhouseData::getRelationshipFqdnAddressIpSql,ReadClickhouseData::getRelationFqdnAddressIpDocument); @@ -60,9 +60,9 @@ public class UpdateGraphData { // VisitIp2Fqdn.class,BaseEdgeDocument.class, // ReadClickhouseData::getRelationshipIpVisitFqdnSql,ReadClickhouseData::getRelationIpVisitFqdnDocument); -// updateDocument(newRelationSubsciberLocateIpMap,historyRelationSubsciberLocateIpMap,"R_LOCATE_SUBSCRIBER2IP", -// LocateSubscriber2Ip.class,BaseEdgeDocument.class, -// ReadClickhouseData::getRelationshipSubsciberLocateIpSql,ReadClickhouseData::getRelationshipSubsciberLocateIpDocument); + updateDocument(newRelationSubsciberLocateIpMap,historyRelationSubsciberLocateIpMap,"R_LOCATE_SUBSCRIBER2IP", + LocateSubscriber2Ip.class,BaseEdgeDocument.class, + ReadClickhouseData::getRelationshipSubsciberLocateIpSql,ReadClickhouseData::getRelationshipSubsciberLocateIpDocument); long last = System.currentTimeMillis(); diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java index d8ded7e..0f1b90e 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java @@ -59,7 +59,7 @@ public class ReadHistoryArangoData extends Thread { switch (table) { case "R_LOCATE_FQDN2IP": updateProtocolDocument(doc); - deleteDistinctClientIpByTime(doc); +// deleteDistinctClientIpByTime(doc); list.add(doc); break; default: diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/update/relationship/LocateFqdn2Ip.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/update/relationship/LocateFqdn2Ip.java index 82d1a1a..c8dca13 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/service/update/relationship/LocateFqdn2Ip.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/update/relationship/LocateFqdn2Ip.java @@ -53,12 +53,12 @@ public class LocateFqdn2Ip extends Relationship { } private void updateDistinctClientIp(BaseEdgeDocument newEdgeDocument,BaseEdgeDocument edgeDocument){ - String[] distCip = (String[]) edgeDocument.getAttribute("DIST_CIP"); - long[] distCipTs = (long[]) edgeDocument.getAttribute("DIST_CIP_TS"); + ArrayList distCip = (ArrayList) edgeDocument.getAttribute("DIST_CIP"); + ArrayList distCipTs = (ArrayList) edgeDocument.getAttribute("DIST_CIP_TS"); HashMap distCipToTs = new HashMap<>(); - if (distCip.length == distCipTs.length){ - for (int i = 0;i < distCip.length;i++){ - distCipToTs.put(distCip[i],distCipTs[i]); + if (distCip.size() == distCipTs.size()){ + for (int i = 0;i < distCip.size();i++){ + distCipToTs.put(distCip.get(i),distCipTs.get(i)); } } Object[] distCipRecent = (Object[])newEdgeDocument.getAttribute("DIST_CIP"); diff --git a/IP-learning-graph/src/main/resources/application.properties b/IP-learning-graph/src/main/resources/application.properties index b9a2c96..e30d8db 100644 --- a/IP-learning-graph/src/main/resources/application.properties +++ b/IP-learning-graph/src/main/resources/application.properties @@ -3,7 +3,7 @@ arangoDB.host=192.168.44.12 arangoDB.port=8529 arangoDB.user=upsert arangoDB.password=ceiec2018 -arangoDB.DB.name=tsg_galaxy_v3 +arangoDB.DB.name=tsg_galaxy_v3_test arangoDB.batch=100000 arangoDB.ttl=3600 @@ -15,9 +15,9 @@ thread.await.termination.time=10 #读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围 -time.limit.type=0 -read.clickhouse.max.time=1598323368 -read.clickhouse.min.time=1597222501 +time.limit.type=1 +read.clickhouse.max.time=1603421554 +read.clickhouse.min.time=1603354682 update.interval=3600 distinct.client.ip.num=100 diff --git a/IP-learning-graph/src/main/resources/clickhouse.properties b/IP-learning-graph/src/main/resources/clickhouse.properties index 3aaf752..f24c51c 100644 --- a/IP-learning-graph/src/main/resources/clickhouse.properties +++ b/IP-learning-graph/src/main/resources/clickhouse.properties @@ -1,6 +1,6 @@ drivers=ru.yandex.clickhouse.ClickHouseDriver mdb.user=default -db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=3600000&compress=0 +db.id=192.168.44.10:8123/tsg_galaxy_v3?socket_timeout=3600000&compress=0 mdb.password=ceiec2019 initialsize=1 minidle=1