gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改日志输出格式

Browse Source
This commit is contained in:
wanglihui
2020-06-29 19:06:23 +08:00
parent e7048e7c53
commit 82a57ff8ec
16 changed files with 98 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseArangoData.java
View File

@@ -1,10 +1,10 @@
package cn.ac.iie.dao;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.etl.ArangoEFqdnAddressIpToMap;
import cn.ac.iie.etl.ArangoEIpVisitFqdnToMap;
import cn.ac.iie.etl.ArangoVFqdnToMap;
import cn.ac.iie.etl.ArangoVIpToMap;
import cn.ac.iie.etl.fqdn2ip.ArangoEFqdnAddressIpToMap;
import cn.ac.iie.etl.ip2fqdn.ArangoEIpVisitFqdnToMap;
import cn.ac.iie.etl.fqdn.ArangoVFqdnToMap;
import cn.ac.iie.etl.ip.ArangoVIpToMap;
import cn.ac.iie.utils.ArangoDBConnect;
import cn.ac.iie.utils.ExecutorThreadPool;
import com.arangodb.ArangoCursor;
@@ -68,7 +68,7 @@ public class BaseArangoData {
long maxTime = 0L;
long diffTime = 0L;
long startTime = System.currentTimeMillis();
LOG.info(sql);
// LOG.info(sql);
ArangoCursor<BaseDocument> timeDoc = arangoDBConnect.executorQuery(sql, BaseDocument.class);
try {
if (timeDoc != null){
@@ -78,7 +78,7 @@ public class BaseArangoData {
minTime = Long.parseLong(doc.getAttribute("min_time").toString());
}
long lastTime = System.currentTimeMillis();
LOG.info("查询最大最小时间用时：" + (lastTime - startTime));
LOG.info(sql+"\n查询最大最小时间用时：" + (lastTime - startTime));
diffTime = (maxTime - minTime) / ApplicationConfig.THREAD_POOL_NUMBER;
}else {
LOG.warn("获取ArangoDb时间范围为空");

10
IP-learning-graph/src/main/java/cn/ac/iie/etl/ArangoVFqdnToMap.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn/ArangoVFqdnToMap.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.fqdn;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoVFqdnToMap implements Runnable {
private long diffTime;
private int threadNumber;
private ArangoVFqdnToMap(){}
public ArangoVFqdnToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
this.arangoDBConnect = arangoDBConnect;
this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoVFqdnToMap implements Runnable {
long maxThreadTime = finalMinTime + (threadNumber + 1)* diffTime;
long minThreadTime = finalMinTime + threadNumber * diffTime;
String query = "FOR doc IN FQDN filter doc.FIRST_FOUND_TIME >= "+minThreadTime+" and doc.FIRST_FOUND_TIME <= "+maxThreadTime+" RETURN doc";
LOG.info(name+":"+query);
// LOG.info(name+":"+query);
long s = System.currentTimeMillis();
ArangoCursor<BaseDocument> docs = arangoDBConnect.executorQuery(query, BaseDocument.class);
@@ -44,9 +42,9 @@ public class ArangoVFqdnToMap implements Runnable {
BaseArangoData.v_Fqdn_Map.put(key,doc);
i++;
}
LOG.info(name+":共处理数据"+ i);
// LOG.info(name+":共处理FQDN数据"+ i);
long l = System.currentTimeMillis();
LOG.info(name+"运行时间："+(l-s));
LOG.info(query+"\n处理FQDN数据"+ i+"条,运行时间："+(l-s));
}else {
LOG.warn("获取VFqdn异常，结果为空");
}

4
IP-learning-graph/src/main/java/cn/ac/iie/etl/UpdateVFqdn.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn/UpdateVFqdn.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.fqdn;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseArangoData;
@@ -8,8 +8,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Set;
public class UpdateVFqdn implements Runnable{
private static final Logger LOG = LoggerFactory.getLogger(UpdateVFqdn.class);

9
IP-learning-graph/src/main/java/cn/ac/iie/etl/ArangoEFqdnAddressIpToMap.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn2ip/ArangoEFqdnAddressIpToMap.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.fqdn2ip;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
private long diffTime;
private int threadNumber;
private ArangoEFqdnAddressIpToMap(){}
public ArangoEFqdnAddressIpToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
this.arangoDBConnect = arangoDBConnect;
this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
long minThreadTime = finalMinTime + threadNumber * diffTime;
String query = "FOR doc IN R_LOCATE_FQDN2IP filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
LOG.info(name + ":" + query);
// LOG.info(name + ":" + query);
long s = System.currentTimeMillis();
try {
ArangoCursor<BaseEdgeDocument> docs = arangoDBConnect.executorQuery(query, BaseEdgeDocument.class);
@@ -43,9 +41,8 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
BaseArangoData.e_Fqdn_Address_Ip_Map.put(key, doc);
i++;
}
LOG.info(name + ":共处理数据" + i);
long l = System.currentTimeMillis();
LOG.info(name + "运行时间：" + (l - s));
LOG.info(query+ "\n处理R_LOCATE_FQDN2IP数据" + i + "条，运行时间：" + (l - s));
}else {
LOG.warn("查询R_LOCATE_FQDN2IP异常,结果为空");
}

2
IP-learning-graph/src/main/java/cn/ac/iie/etl/UpdateEFqdnAddressIp.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn2ip/UpdateEFqdnAddressIp.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.fqdn2ip;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseArangoData;

9
IP-learning-graph/src/main/java/cn/ac/iie/etl/ArangoVIpToMap.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/ip/ArangoVIpToMap.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.ip;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoVIpToMap implements Runnable {
private long diffTime;
private int threadNumber;
private ArangoVIpToMap() {}
public ArangoVIpToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
this.arangoDBConnect = arangoDBConnect;
this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoVIpToMap implements Runnable {
long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
long minThreadTime = finalMinTime + threadNumber * diffTime;
String query = "FOR doc IN IP filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
LOG.info(name + ":" + query);
// LOG.info(name + ":" + query);
long s = System.currentTimeMillis();
try {
ArangoCursor<BaseDocument> docs = arangoDBConnect.executorQuery(query, BaseDocument.class);
@@ -43,9 +41,8 @@ public class ArangoVIpToMap implements Runnable {
BaseArangoData.v_Ip_Map.put(key, doc);
i++;
}
LOG.info(name + ":共处理数据" + i);
long l = System.currentTimeMillis();
LOG.info(name + "运行时间：" + (l - s));
LOG.info(query+ "\n处理IP数据" + i + "条，运行时间：" + (l - s));
}else {
LOG.warn("获取VIP异常，结果为空");
}

2
IP-learning-graph/src/main/java/cn/ac/iie/etl/UpdateVIP.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/ip/UpdateVIP.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.ip;
import cn.ac.iie.config.ApplicationConfig;

9
IP-learning-graph/src/main/java/cn/ac/iie/etl/ArangoEIpVisitFqdnToMap.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/ip2fqdn/ArangoEIpVisitFqdnToMap.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.ip2fqdn;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
@@ -16,8 +16,6 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
private long diffTime;
private int threadNumber;
private ArangoEIpVisitFqdnToMap(){}
public ArangoEIpVisitFqdnToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
this.arangoDBConnect = arangoDBConnect;
this.finalMinTime = finalMinTime;
@@ -30,7 +28,7 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
long minThreadTime = finalMinTime + threadNumber * diffTime;
String query = "FOR doc IN R_VISIT_IP2FQDN filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
LOG.info(name + ":" + query);
// LOG.info(name + ":" + query);
long s = System.currentTimeMillis();
ArangoCursor<BaseEdgeDocument> docs = arangoDBConnect.executorQuery(query, BaseEdgeDocument.class);
@@ -42,9 +40,8 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
BaseArangoData.e_Ip_Visit_Fqdn_Map.put(key, doc);
i++;
}
LOG.info(name + ":共处理数据" + i);
long l = System.currentTimeMillis();
LOG.info(name + "运行时间：" + (l - s));
LOG.info(query+ "\n处理R_VISIT_IP2FQDN数据" + i + "条，运行时间：" + (l - s));
}else {
LOG.warn("查询R_VISIT_IP2FQDN异常，结果为空");
}

3
IP-learning-graph/src/main/java/cn/ac/iie/etl/UpdateEIpVisitFqdn.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/ip2fqdn/UpdateEIpVisitFqdn.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.ip2fqdn;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseArangoData;
@@ -8,7 +8,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Set;

5
IP-learning-graph/src/main/java/cn/ac/iie/pojo/VertexFqdn.java Normal file
View File

@@ -0,0 +1,5 @@
package cn.ac.iie.pojo;
public class VertexFqdn {
}

1
IP-learning-graph/src/main/java/cn/ac/iie/test/IpLearningApplicationTest.java
View File

@@ -58,7 +58,6 @@ public class IpLearningApplicationTest {
LOG.error("主线程阻塞异常：\n"+e.toString());
}
// BaseClickhouseData.BaseEFqdnAddressIp();
long lastC = System.currentTimeMillis();
LOG.info("更新ArangoDb时间："+(lastC - startC));
}catch (Exception e){

13
IP-learning-graph/src/main/java/cn/ac/iie/utils/ArangoDBConnect.java
View File

@@ -42,7 +42,7 @@ public class ArangoDBConnect {
return conn;
}
public ArangoDatabase getDatabase(){
private ArangoDatabase getDatabase(){
return arangoDB.db(ApplicationConfig.ARANGODB_DB_NAME);
}
@@ -59,7 +59,8 @@ public class ArangoDBConnect {
public <T> ArangoCursor<T> executorQuery(String query,Class<T> type){
ArangoDatabase database = getDatabase();
Map<String, Object> bindVars = new MapBuilder().get();
AqlQueryOptions options = new AqlQueryOptions().ttl(ApplicationConfig.ARANGODB_TTL);
AqlQueryOptions options = new AqlQueryOptions()
.ttl(ApplicationConfig.ARANGODB_TTL);
try {
return database.query(query, bindVars, options, type);
}catch (Exception e){
@@ -70,6 +71,7 @@ public class ArangoDBConnect {
}
}
@Deprecated
public <T> void insertAndUpdate(ArrayList<T> docInsert,ArrayList<T> docUpdate,String collectionName){
ArangoDatabase database = getDatabase();
try {
@@ -97,17 +99,16 @@ public class ArangoDBConnect {
if (!docOverwrite.isEmpty()){
DocumentCreateOptions documentCreateOptions = new DocumentCreateOptions();
documentCreateOptions.overwrite(true);
documentCreateOptions.returnNew(true);
documentCreateOptions.returnOld(true);
documentCreateOptions.silent(true);
MultiDocumentEntity<DocumentCreateEntity<T>> documentCreateEntityMultiDocumentEntity = collection.insertDocuments(docOverwrite, documentCreateOptions);
Collection<ErrorEntity> errors = documentCreateEntityMultiDocumentEntity.getErrors();
for (ErrorEntity errorEntity:errors){
LOG.error("写入arangoDB异常："+errorEntity.getErrorMessage());
}
}
}catch (Exception e){
LOG.error(e.toString());
LOG.error("更新失败："+e.toString());
// clean();
}finally {
docOverwrite.clear();
}

2
IP-learning-graph/src/main/resources/application.properties
View File

@@ -1,5 +1,5 @@
#arangoDB参数配置
arangoDB.host=192.168.40.182
arangoDB.host=192.168.40.127
arangoDB.port=8529
arangoDB.user=root
arangoDB.password=111111