gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加SUBSCRIBER点与R_LOCATE_SUBSCRIBER2IP关系

Browse Source
This commit is contained in:
wanglihui
2020-07-09 17:41:43 +08:00
parent 0e926aa7d0
commit d214e86a41
15 changed files with 263 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java
View File

@@ -1,10 +1,12 @@
package cn.ac.iie.dao;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.etl.relationship.LocateFqdn2Ip;
import cn.ac.iie.etl.relationship.VisitIp2Fqdn;
import cn.ac.iie.etl.vertex.Fqdn;
import cn.ac.iie.etl.vertex.Ip;
import cn.ac.iie.service.relationship.LocateFqdn2Ip;
import cn.ac.iie.service.relationship.LocateSubscriber2Ip;
import cn.ac.iie.service.relationship.VisitIp2Fqdn;
import cn.ac.iie.service.vertex.Fqdn;
import cn.ac.iie.service.vertex.Ip;
import cn.ac.iie.service.vertex.Subscriber;
import cn.ac.iie.utils.ArangoDBConnect;
import cn.ac.iie.utils.ExecutorThreadPool;
import com.arangodb.entity.BaseDocument;
@@ -16,6 +18,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.concurrent.CountDownLatch;
/**
* 更新图数据库业务类
*/
public class UpdateGraphData {
private static final Logger LOG = LoggerFactory.getLogger(UpdateGraphData.class);
private static ExecutorThreadPool pool = ExecutorThreadPool.getInstance();
@@ -38,10 +43,16 @@ public class UpdateGraphData {
baseClickhouseData.BaseEIpVisitFqdn();
updateRelationIpVisitFqdn();
baseClickhouseData.BaseVertexSubscriber();
updateVertexSubscriber();
baseClickhouseData.BaseRelationshipSubscriberLocateIp();
updateRelationshipSubsciberLocateIp();
}catch (Exception e){
e.printStackTrace();
}finally {
ArangoDBConnect.clean();
arangoManger.clean();
}
long lastC = System.currentTimeMillis();
LOG.info("更新ArangoDb时间："+(lastC - startC));
@@ -63,6 +74,38 @@ public class UpdateGraphData {
}
}
private void updateVertexSubscriber(){
try {
countDownLatch = new CountDownLatch(ApplicationConfig.THREAD_POOL_NUMBER);
for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++) {
HashMap<String, ArrayList<BaseDocument>> stringArrayListHashMap = BaseClickhouseData.vSubscriberMap.get(i);
LOG.info("vSubscriber baseDocumentHashMap大小：" + stringArrayListHashMap.size());
Subscriber updateVSubscriber = new Subscriber(stringArrayListHashMap, arangoManger, "SUBSCRIBER", BaseArangoData.v_Subscriber_Map,countDownLatch);
updateVSubscriber.run();
}
countDownLatch.await();
LOG.info("---------SUBSCRIBER vertex 更新完毕---------");
}catch (Exception e){
e.printStackTrace();
}
}
private void updateRelationshipSubsciberLocateIp(){
try {
countDownLatch = new CountDownLatch(ApplicationConfig.THREAD_POOL_NUMBER);
for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++) {
HashMap<String, HashMap<String, BaseEdgeDocument>> baseDocumentHashMap = BaseClickhouseData.eSubsciberLocateIpMap.get(i);
LOG.info("ESubsciberLocateIp baseDocumentHashMap大小：" + baseDocumentHashMap.size());
LocateSubscriber2Ip rLocateSubscriber2IP = new LocateSubscriber2Ip(baseDocumentHashMap, arangoManger, "R_LOCATE_SUBSCRIBER2IP", BaseArangoData.e_Subsciber_Locate_Ip_Map, countDownLatch);
rLocateSubscriber2IP.run();
}
countDownLatch.await();
LOG.info("------------R_LOCATE_SUBSCRIBER2IP relationship 更新完毕----------------");
}catch (Exception e){
e.printStackTrace();
}
}
private void updateVertexIp(){
try {
countDownLatch = new CountDownLatch(ApplicationConfig.THREAD_POOL_NUMBER);