diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/config/ApplicationConfig.java b/IP-learning-graph/src/main/java/cn/ac/iie/config/ApplicationConfig.java index 68f8398..8b9b405 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/config/ApplicationConfig.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/config/ApplicationConfig.java @@ -14,7 +14,7 @@ public class ApplicationConfig { public static final Integer ARANGODB_BATCH = ConfigUtils.getIntProperty( "arangoDB.batch"); public static final Integer UPDATE_ARANGO_BATCH = ConfigUtils.getIntProperty("update.arango.batch"); - public static final String ARANGODB_READ_LIMIT = ConfigUtils.getStringProperty("arangoDB.read.limit"); + public static final Long ARANGODB_READ_LIMIT = ConfigUtils.getLongProperty("arangoDB.read.limit"); public static final Integer THREAD_POOL_NUMBER = ConfigUtils.getIntProperty( "thread.pool.number"); public static final Integer THREAD_AWAIT_TERMINATION_TIME = ConfigUtils.getIntProperty( "thread.await.termination.time"); diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseArangoData.java b/IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseArangoData.java index 841c32b..22c934e 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseArangoData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseArangoData.java @@ -78,41 +78,12 @@ public class BaseArangoData { private String getQuerySql(Long cnt,int threadNumber, String table){ long sepNum = cnt / ApplicationConfig.THREAD_POOL_NUMBER + 1; long offsetNum = threadNumber * sepNum; + if (sepNum >= ApplicationConfig.ARANGODB_READ_LIMIT * 10000){ + sepNum = ApplicationConfig.ARANGODB_READ_LIMIT * 10000; + } return "FOR doc IN " + table + " limit "+offsetNum+","+sepNum+" RETURN doc"; } - private long[] getTimeRange(String table){ - long minTime = 0L; - long maxTime = 0L; - long startTime = System.currentTimeMillis(); - String sql = "LET doc = (FOR doc IN "+table+" RETURN doc) return {max_time:MAX(doc[*].FIRST_FOUND_TIME),min_time:MIN(doc[*].FIRST_FOUND_TIME)}"; - ArangoCursor timeDoc = arangoDBConnect.executorQuery(sql, BaseDocument.class); - try { - if (timeDoc != null){ - while (timeDoc.hasNext()) { - BaseDocument doc = timeDoc.next(); - maxTime = Long.parseLong(doc.getAttribute("max_time").toString()) + ApplicationConfig.THREAD_POOL_NUMBER; - minTime = Long.parseLong(doc.getAttribute("min_time").toString()); - } - long lastTime = System.currentTimeMillis(); - LOG.info(sql+"\n查询最大最小时间用时：" + (lastTime - startTime)); - }else { - LOG.warn("获取ArangoDb时间范围为空"); - } - }catch (Exception e){ - e.printStackTrace(); - } - return new long[]{minTime, maxTime}; - } - - private String getQuerySql(long[] timeRange,int threadNumber,String table){ - long minTime = timeRange[0]; - long maxTime = timeRange[1]; - long diffTime = (maxTime - minTime) / ApplicationConfig.THREAD_POOL_NUMBER; - long maxThreadTime = minTime + (threadNumber + 1)* diffTime; - long minThreadTime = minTime + threadNumber * diffTime; - return "FOR doc IN "+table+" filter doc.FIRST_FOUND_TIME >= "+minThreadTime+" and doc.FIRST_FOUND_TIME <= "+maxThreadTime+" " + ApplicationConfig.ARANGODB_READ_LIMIT + " RETURN doc"; - } } diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java b/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java index 12fc1bd..b9ead95 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java @@ -44,14 +44,14 @@ public class UpdateGraphData { long start = System.currentTimeMillis(); try { - updateDocument(newVertexFqdnMap, historyVertexFqdnMap, "FQDN", Fqdn.class,BaseDocument.class, - ReadClickhouseData::getVertexFqdnSql,ReadClickhouseData::getVertexFqdnDocument); +// updateDocument(newVertexFqdnMap, historyVertexFqdnMap, "FQDN", Fqdn.class,BaseDocument.class, +// ReadClickhouseData::getVertexFqdnSql,ReadClickhouseData::getVertexFqdnDocument); - updateDocument(newVertexIpMap,historyVertexIpMap,"IP", Ip.class,BaseDocument.class, - ReadClickhouseData::getVertexIpSql,ReadClickhouseData::getVertexIpDocument); +// updateDocument(newVertexIpMap,historyVertexIpMap,"IP", Ip.class,BaseDocument.class, +// ReadClickhouseData::getVertexIpSql,ReadClickhouseData::getVertexIpDocument); - updateDocument(newVertexSubscriberMap,historyVertexSubscriberMap,"SUBSCRIBER", Subscriber.class,BaseDocument.class, - ReadClickhouseData::getVertexSubscriberSql,ReadClickhouseData::getVertexSubscriberDocument); +// updateDocument(newVertexSubscriberMap,historyVertexSubscriberMap,"SUBSCRIBER", Subscriber.class,BaseDocument.class, +// ReadClickhouseData::getVertexSubscriberSql,ReadClickhouseData::getVertexSubscriberDocument); updateDocument(newRelationFqdnAddressIpMap,historyRelationFqdnAddressIpMap,"R_LOCATE_FQDN2IP", LocateFqdn2Ip.class,BaseEdgeDocument.class, ReadClickhouseData::getRelationshipFqdnAddressIpSql,ReadClickhouseData::getRelationFqdnAddressIpDocument); @@ -60,9 +60,9 @@ public class UpdateGraphData { // VisitIp2Fqdn.class,BaseEdgeDocument.class, // ReadClickhouseData::getRelationshipIpVisitFqdnSql,ReadClickhouseData::getRelationIpVisitFqdnDocument); - updateDocument(newRelationSubsciberLocateIpMap,historyRelationSubsciberLocateIpMap,"R_LOCATE_SUBSCRIBER2IP", - LocateSubscriber2Ip.class,BaseEdgeDocument.class, - ReadClickhouseData::getRelationshipSubsciberLocateIpSql,ReadClickhouseData::getRelationshipSubsciberLocateIpDocument); +// updateDocument(newRelationSubsciberLocateIpMap,historyRelationSubsciberLocateIpMap,"R_LOCATE_SUBSCRIBER2IP", +// LocateSubscriber2Ip.class,BaseEdgeDocument.class, +// ReadClickhouseData::getRelationshipSubsciberLocateIpSql,ReadClickhouseData::getRelationshipSubsciberLocateIpDocument); long last = System.currentTimeMillis(); diff --git a/IP-learning-graph/src/main/resources/application.properties b/IP-learning-graph/src/main/resources/application.properties index fec32ff..b9a2c96 100644 --- a/IP-learning-graph/src/main/resources/application.properties +++ b/IP-learning-graph/src/main/resources/application.properties @@ -7,10 +7,10 @@ arangoDB.DB.name=tsg_galaxy_v3 arangoDB.batch=100000 arangoDB.ttl=3600 -arangoDB.read.limit= +arangoDB.read.limit=10 update.arango.batch=10000 -thread.pool.number=10 +thread.pool.number=40 thread.await.termination.time=10 @@ -20,6 +20,6 @@ read.clickhouse.max.time=1598323368 read.clickhouse.min.time=1597222501 update.interval=3600 -distinct.client.ip.num=10000 +distinct.client.ip.num=100 recent.count.hour=24 diff --git a/IP-learning-graph/src/main/resources/clickhouse.properties b/IP-learning-graph/src/main/resources/clickhouse.properties index 4027174..3aaf752 100644 --- a/IP-learning-graph/src/main/resources/clickhouse.properties +++ b/IP-learning-graph/src/main/resources/clickhouse.properties @@ -1,7 +1,6 @@ drivers=ru.yandex.clickhouse.ClickHouseDriver mdb.user=default -#db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000 -db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0 +db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=3600000&compress=0 mdb.password=ceiec2019 initialsize=1 minidle=1 diff --git a/IP-learning-graph/src/main/resources/log4j.properties b/IP-learning-graph/src/main/resources/log4j.properties index 2366506..21cea3d 100644 --- a/IP-learning-graph/src/main/resources/log4j.properties +++ b/IP-learning-graph/src/main/resources/log4j.properties @@ -17,7 +17,6 @@ log4j.appender.file.encoding=UTF-8 log4j.appender.file.Append=true #··زӦĿ #log4j.appender.file.file=/home/ceiec/iplearning/logs/ip-learning-application.log -#log4j.appender.file.file=/home/ceiec/iplearning/testLog/ip-learning-application.log log4j.appender.file.file=./logs/ip-learning-application.log log4j.appender.file.DatePattern='.'yyyy-MM-dd log4j.appender.file.layout=org.apache.log4j.PatternLayout