gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

client IP只根据阈值统计，去掉最近24小时限制

Browse Source
This commit is contained in:
wanglihui
2020-10-23 14:04:53 +08:00
parent 5cb7327f7b
commit e4d7737725
5 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java
View File

@@ -47,11 +47,11 @@ public class UpdateGraphData {
// updateDocument(newVertexFqdnMap, historyVertexFqdnMap, "FQDN", Fqdn.class,BaseDocument.class, // updateDocument(newVertexFqdnMap, historyVertexFqdnMap, "FQDN", Fqdn.class,BaseDocument.class,
// ReadClickhouseData::getVertexFqdnSql,ReadClickhouseData::getVertexFqdnDocument); // ReadClickhouseData::getVertexFqdnSql,ReadClickhouseData::getVertexFqdnDocument);
// updateDocument(newVertexIpMap,historyVertexIpMap,"IP", Ip.class,BaseDocument.class, updateDocument(newVertexIpMap,historyVertexIpMap,"IP", Ip.class,BaseDocument.class,
// ReadClickhouseData::getVertexIpSql,ReadClickhouseData::getVertexIpDocument); ReadClickhouseData::getVertexIpSql,ReadClickhouseData::getVertexIpDocument);
// updateDocument(newVertexSubscriberMap,historyVertexSubscriberMap,"SUBSCRIBER", Subscriber.class,BaseDocument.class, updateDocument(newVertexSubscriberMap,historyVertexSubscriberMap,"SUBSCRIBER", Subscriber.class,BaseDocument.class,
// ReadClickhouseData::getVertexSubscriberSql,ReadClickhouseData::getVertexSubscriberDocument); ReadClickhouseData::getVertexSubscriberSql,ReadClickhouseData::getVertexSubscriberDocument);
updateDocument(newRelationFqdnAddressIpMap,historyRelationFqdnAddressIpMap,"R_LOCATE_FQDN2IP", LocateFqdn2Ip.class,BaseEdgeDocument.class, updateDocument(newRelationFqdnAddressIpMap,historyRelationFqdnAddressIpMap,"R_LOCATE_FQDN2IP", LocateFqdn2Ip.class,BaseEdgeDocument.class,
ReadClickhouseData::getRelationshipFqdnAddressIpSql,ReadClickhouseData::getRelationFqdnAddressIpDocument); ReadClickhouseData::getRelationshipFqdnAddressIpSql,ReadClickhouseData::getRelationFqdnAddressIpDocument);
@@ -60,9 +60,9 @@ public class UpdateGraphData {
// VisitIp2Fqdn.class,BaseEdgeDocument.class, // VisitIp2Fqdn.class,BaseEdgeDocument.class,
// ReadClickhouseData::getRelationshipIpVisitFqdnSql,ReadClickhouseData::getRelationIpVisitFqdnDocument); // ReadClickhouseData::getRelationshipIpVisitFqdnSql,ReadClickhouseData::getRelationIpVisitFqdnDocument);
// updateDocument(newRelationSubsciberLocateIpMap,historyRelationSubsciberLocateIpMap,"R_LOCATE_SUBSCRIBER2IP", updateDocument(newRelationSubsciberLocateIpMap,historyRelationSubsciberLocateIpMap,"R_LOCATE_SUBSCRIBER2IP",
// LocateSubscriber2Ip.class,BaseEdgeDocument.class, LocateSubscriber2Ip.class,BaseEdgeDocument.class,
// ReadClickhouseData::getRelationshipSubsciberLocateIpSql,ReadClickhouseData::getRelationshipSubsciberLocateIpDocument); ReadClickhouseData::getRelationshipSubsciberLocateIpSql,ReadClickhouseData::getRelationshipSubsciberLocateIpDocument);
long last = System.currentTimeMillis(); long last = System.currentTimeMillis();

2
IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java
View File

@@ -59,7 +59,7 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
switch (table) { switch (table) {
case "R_LOCATE_FQDN2IP": case "R_LOCATE_FQDN2IP":
updateProtocolDocument(doc); updateProtocolDocument(doc);
deleteDistinctClientIpByTime(doc); // deleteDistinctClientIpByTime(doc);
list.add(doc); list.add(doc);
break; break;
default: default:

10
IP-learning-graph/src/main/java/cn/ac/iie/service/update/relationship/LocateFqdn2Ip.java
View File

@@ -53,12 +53,12 @@ public class LocateFqdn2Ip extends Relationship {
} }
private void updateDistinctClientIp(BaseEdgeDocument newEdgeDocument,BaseEdgeDocument edgeDocument){ private void updateDistinctClientIp(BaseEdgeDocument newEdgeDocument,BaseEdgeDocument edgeDocument){
String[] distCip = (String[]) edgeDocument.getAttribute("DIST_CIP"); ArrayList<String> distCip = (ArrayList<String>) edgeDocument.getAttribute("DIST_CIP");
long[] distCipTs = (long[]) edgeDocument.getAttribute("DIST_CIP_TS"); ArrayList<Long> distCipTs = (ArrayList<Long>) edgeDocument.getAttribute("DIST_CIP_TS");
HashMap<String, Long> distCipToTs = new HashMap<>(); HashMap<String, Long> distCipToTs = new HashMap<>();
if (distCip.length == distCipTs.length){ if (distCip.size() == distCipTs.size()){
for (int i = 0;i < distCip.length;i++){ for (int i = 0;i < distCip.size();i++){
distCipToTs.put(distCip[i],distCipTs[i]); distCipToTs.put(distCip.get(i),distCipTs.get(i));
} }
} }
Object[] distCipRecent = (Object[])newEdgeDocument.getAttribute("DIST_CIP"); Object[] distCipRecent = (Object[])newEdgeDocument.getAttribute("DIST_CIP");

8
IP-learning-graph/src/main/resources/application.properties
View File

@@ -3,7 +3,7 @@ arangoDB.host=192.168.44.12
arangoDB.port=8529 arangoDB.port=8529
arangoDB.user=upsert arangoDB.user=upsert
arangoDB.password=ceiec2018 arangoDB.password=ceiec2018
arangoDB.DB.name=tsg_galaxy_v3 arangoDB.DB.name=tsg_galaxy_v3_test
arangoDB.batch=100000 arangoDB.batch=100000
arangoDB.ttl=3600 arangoDB.ttl=3600
@@ -15,9 +15,9 @@ thread.await.termination.time=10
#读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围 #读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围
time.limit.type=0 time.limit.type=1
read.clickhouse.max.time=1598323368 read.clickhouse.max.time=1603421554
read.clickhouse.min.time=1597222501 read.clickhouse.min.time=1603354682
update.interval=3600 update.interval=3600
distinct.client.ip.num=100 distinct.client.ip.num=100

2
IP-learning-graph/src/main/resources/clickhouse.properties
View File

@@ -1,6 +1,6 @@
drivers=ru.yandex.clickhouse.ClickHouseDriver drivers=ru.yandex.clickhouse.ClickHouseDriver
mdb.user=default mdb.user=default
db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=3600000&compress=0 db.id=192.168.44.10:8123/tsg_galaxy_v3?socket_timeout=3600000&compress=0
mdb.password=ceiec2019 mdb.password=ceiec2019
initialsize=1 initialsize=1
minidle=1 minidle=1