gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改日志输出格式

Browse Source
This commit is contained in:
wanglihui
2020-06-29 19:06:23 +08:00
parent e7048e7c53
commit 82a57ff8ec
16 changed files with 98 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
IP-learning-graph/pom.xml
View File

@@ -42,7 +42,8 @@
<dependency>
<groupId>com.arangodb</groupId>
<artifactId>arangodb-java-driver</artifactId>
<version>5.0.4</version>
<!--<version>5.0.4</version>-->
<version>6.6.3</version>
</dependency>
</dependencies>

12
IP-learning-graph/src/main/java/cn/ac/iie/dao/BaseArangoData.java
View File

@@ -1,10 +1,10 @@
package cn.ac.iie.dao;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.etl.ArangoEFqdnAddressIpToMap;
import cn.ac.iie.etl.ArangoEIpVisitFqdnToMap;
import cn.ac.iie.etl.ArangoVFqdnToMap;
import cn.ac.iie.etl.ArangoVIpToMap;
import cn.ac.iie.etl.fqdn2ip.ArangoEFqdnAddressIpToMap;
import cn.ac.iie.etl.ip2fqdn.ArangoEIpVisitFqdnToMap;
import cn.ac.iie.etl.fqdn.ArangoVFqdnToMap;
import cn.ac.iie.etl.ip.ArangoVIpToMap;
import cn.ac.iie.utils.ArangoDBConnect;
import cn.ac.iie.utils.ExecutorThreadPool;
import com.arangodb.ArangoCursor;
@@ -68,7 +68,7 @@ public class BaseArangoData {
long maxTime = 0L;
long diffTime = 0L;
long startTime = System.currentTimeMillis();
LOG.info(sql);
// LOG.info(sql);
ArangoCursor<BaseDocument> timeDoc = arangoDBConnect.executorQuery(sql, BaseDocument.class);
try {
if (timeDoc != null){
@@ -78,7 +78,7 @@ public class BaseArangoData {
minTime = Long.parseLong(doc.getAttribute("min_time").toString());
}
long lastTime = System.currentTimeMillis();
LOG.info("查询最大最小时间用时：" + (lastTime - startTime));
LOG.info(sql+"\n查询最大最小时间用时：" + (lastTime - startTime));
diffTime = (maxTime - minTime) / ApplicationConfig.THREAD_POOL_NUMBER;
}else {
LOG.warn("获取ArangoDb时间范围为空");

10
IP-learning-graph/src/main/java/cn/ac/iie/etl/ArangoVFqdnToMap.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn/ArangoVFqdnToMap.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.fqdn;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoVFqdnToMap implements Runnable {
private long diffTime;
private int threadNumber;
private ArangoVFqdnToMap(){}
public ArangoVFqdnToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
this.arangoDBConnect = arangoDBConnect;
this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoVFqdnToMap implements Runnable {
long maxThreadTime = finalMinTime + (threadNumber + 1)* diffTime;
long minThreadTime = finalMinTime + threadNumber * diffTime;
String query = "FOR doc IN FQDN filter doc.FIRST_FOUND_TIME >= "+minThreadTime+" and doc.FIRST_FOUND_TIME <= "+maxThreadTime+" RETURN doc";
LOG.info(name+":"+query);
// LOG.info(name+":"+query);
long s = System.currentTimeMillis();
ArangoCursor<BaseDocument> docs = arangoDBConnect.executorQuery(query, BaseDocument.class);
@@ -44,9 +42,9 @@ public class ArangoVFqdnToMap implements Runnable {
BaseArangoData.v_Fqdn_Map.put(key,doc);
i++;
}
LOG.info(name+":共处理数据"+ i);
// LOG.info(name+":共处理FQDN数据"+ i);
long l = System.currentTimeMillis();
LOG.info(name+"运行时间："+(l-s));
LOG.info(query+"\n处理FQDN数据"+ i+"条,运行时间："+(l-s));
}else {
LOG.warn("获取VFqdn异常，结果为空");
}

4
IP-learning-graph/src/main/java/cn/ac/iie/etl/UpdateVFqdn.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn/UpdateVFqdn.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.fqdn;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseArangoData;
@@ -8,8 +8,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Set;
public class UpdateVFqdn implements Runnable{
private static final Logger LOG = LoggerFactory.getLogger(UpdateVFqdn.class);

9
IP-learning-graph/src/main/java/cn/ac/iie/etl/ArangoEFqdnAddressIpToMap.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn2ip/ArangoEFqdnAddressIpToMap.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.fqdn2ip;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
private long diffTime;
private int threadNumber;
private ArangoEFqdnAddressIpToMap(){}
public ArangoEFqdnAddressIpToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
this.arangoDBConnect = arangoDBConnect;
this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
long minThreadTime = finalMinTime + threadNumber * diffTime;
String query = "FOR doc IN R_LOCATE_FQDN2IP filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
LOG.info(name + ":" + query);
// LOG.info(name + ":" + query);
long s = System.currentTimeMillis();
try {
ArangoCursor<BaseEdgeDocument> docs = arangoDBConnect.executorQuery(query, BaseEdgeDocument.class);
@@ -43,9 +41,8 @@ public class ArangoEFqdnAddressIpToMap implements Runnable{
BaseArangoData.e_Fqdn_Address_Ip_Map.put(key, doc);
i++;
}
LOG.info(name + ":共处理数据" + i);
long l = System.currentTimeMillis();
LOG.info(name + "运行时间：" + (l - s));
LOG.info(query+ "\n处理R_LOCATE_FQDN2IP数据" + i + "条，运行时间：" + (l - s));
}else {
LOG.warn("查询R_LOCATE_FQDN2IP异常,结果为空");
}

2
IP-learning-graph/src/main/java/cn/ac/iie/etl/UpdateEFqdnAddressIp.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/fqdn2ip/UpdateEFqdnAddressIp.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.fqdn2ip;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseArangoData;

9
IP-learning-graph/src/main/java/cn/ac/iie/etl/ArangoVIpToMap.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/ip/ArangoVIpToMap.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.ip;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
@@ -17,8 +17,6 @@ public class ArangoVIpToMap implements Runnable {
private long diffTime;
private int threadNumber;
private ArangoVIpToMap() {}
public ArangoVIpToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
this.arangoDBConnect = arangoDBConnect;
this.finalMinTime = finalMinTime;
@@ -31,7 +29,7 @@ public class ArangoVIpToMap implements Runnable {
long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
long minThreadTime = finalMinTime + threadNumber * diffTime;
String query = "FOR doc IN IP filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
LOG.info(name + ":" + query);
// LOG.info(name + ":" + query);
long s = System.currentTimeMillis();
try {
ArangoCursor<BaseDocument> docs = arangoDBConnect.executorQuery(query, BaseDocument.class);
@@ -43,9 +41,8 @@ public class ArangoVIpToMap implements Runnable {
BaseArangoData.v_Ip_Map.put(key, doc);
i++;
}
LOG.info(name + ":共处理数据" + i);
long l = System.currentTimeMillis();
LOG.info(name + "运行时间：" + (l - s));
LOG.info(query+ "\n处理IP数据" + i + "条，运行时间：" + (l - s));
}else {
LOG.warn("获取VIP异常，结果为空");
}

2
IP-learning-graph/src/main/java/cn/ac/iie/etl/UpdateVIP.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/ip/UpdateVIP.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.ip;
import cn.ac.iie.config.ApplicationConfig;

9
IP-learning-graph/src/main/java/cn/ac/iie/etl/ArangoEIpVisitFqdnToMap.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/ip2fqdn/ArangoEIpVisitFqdnToMap.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.ip2fqdn;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
@@ -16,8 +16,6 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
private long diffTime;
private int threadNumber;
private ArangoEIpVisitFqdnToMap(){}
public ArangoEIpVisitFqdnToMap(ArangoDBConnect arangoDBConnect, long finalMinTime, long diffTime, int threadNumber) {
this.arangoDBConnect = arangoDBConnect;
this.finalMinTime = finalMinTime;
@@ -30,7 +28,7 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
long maxThreadTime = finalMinTime + (threadNumber + 1) * diffTime;
long minThreadTime = finalMinTime + threadNumber * diffTime;
String query = "FOR doc IN R_VISIT_IP2FQDN filter doc.FIRST_FOUND_TIME >= " + minThreadTime + " and doc.FIRST_FOUND_TIME <= " + maxThreadTime + " RETURN doc";
LOG.info(name + ":" + query);
// LOG.info(name + ":" + query);
long s = System.currentTimeMillis();
ArangoCursor<BaseEdgeDocument> docs = arangoDBConnect.executorQuery(query, BaseEdgeDocument.class);
@@ -42,9 +40,8 @@ public class ArangoEIpVisitFqdnToMap implements Runnable {
BaseArangoData.e_Ip_Visit_Fqdn_Map.put(key, doc);
i++;
}
LOG.info(name + ":共处理数据" + i);
long l = System.currentTimeMillis();
LOG.info(name + "运行时间：" + (l - s));
LOG.info(query+ "\n处理R_VISIT_IP2FQDN数据" + i + "条，运行时间：" + (l - s));
}else {
LOG.warn("查询R_VISIT_IP2FQDN异常，结果为空");
}

3
IP-learning-graph/src/main/java/cn/ac/iie/etl/UpdateEIpVisitFqdn.java → IP-learning-graph/src/main/java/cn/ac/iie/etl/ip2fqdn/UpdateEIpVisitFqdn.java
View File

@@ -1,4 +1,4 @@
package cn.ac.iie.etl;
package cn.ac.iie.etl.ip2fqdn;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseArangoData;
@@ -8,7 +8,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Set;

5
IP-learning-graph/src/main/java/cn/ac/iie/pojo/VertexFqdn.java Normal file
View File

@@ -0,0 +1,5 @@
package cn.ac.iie.pojo;
public class VertexFqdn {
}

1
IP-learning-graph/src/main/java/cn/ac/iie/test/IpLearningApplicationTest.java
View File

@@ -58,7 +58,6 @@ public class IpLearningApplicationTest {
LOG.error("主线程阻塞异常：\n"+e.toString());
}
// BaseClickhouseData.BaseEFqdnAddressIp();
long lastC = System.currentTimeMillis();
LOG.info("更新ArangoDb时间："+(lastC - startC));
}catch (Exception e){

13
IP-learning-graph/src/main/java/cn/ac/iie/utils/ArangoDBConnect.java
View File

@@ -42,7 +42,7 @@ public class ArangoDBConnect {
return conn;
}
public ArangoDatabase getDatabase(){
private ArangoDatabase getDatabase(){
return arangoDB.db(ApplicationConfig.ARANGODB_DB_NAME);
}
@@ -59,7 +59,8 @@ public class ArangoDBConnect {
public <T> ArangoCursor<T> executorQuery(String query,Class<T> type){
ArangoDatabase database = getDatabase();
Map<String, Object> bindVars = new MapBuilder().get();
AqlQueryOptions options = new AqlQueryOptions().ttl(ApplicationConfig.ARANGODB_TTL);
AqlQueryOptions options = new AqlQueryOptions()
.ttl(ApplicationConfig.ARANGODB_TTL);
try {
return database.query(query, bindVars, options, type);
}catch (Exception e){
@@ -70,6 +71,7 @@ public class ArangoDBConnect {
}
}
@Deprecated
public <T> void insertAndUpdate(ArrayList<T> docInsert,ArrayList<T> docUpdate,String collectionName){
ArangoDatabase database = getDatabase();
try {
@@ -97,17 +99,16 @@ public class ArangoDBConnect {
if (!docOverwrite.isEmpty()){
DocumentCreateOptions documentCreateOptions = new DocumentCreateOptions();
documentCreateOptions.overwrite(true);
documentCreateOptions.returnNew(true);
documentCreateOptions.returnOld(true);
documentCreateOptions.silent(true);
MultiDocumentEntity<DocumentCreateEntity<T>> documentCreateEntityMultiDocumentEntity = collection.insertDocuments(docOverwrite, documentCreateOptions);
Collection<ErrorEntity> errors = documentCreateEntityMultiDocumentEntity.getErrors();
for (ErrorEntity errorEntity:errors){
LOG.error("写入arangoDB异常："+errorEntity.getErrorMessage());
}
}
}catch (Exception e){
LOG.error(e.toString());
LOG.error("更新失败："+e.toString());
// clean();
}finally {
docOverwrite.clear();
}

2
IP-learning-graph/src/main/resources/application.properties
View File

@@ -1,5 +1,5 @@
#arangoDB参数配置
arangoDB.host=192.168.40.182
arangoDB.host=192.168.40.127
arangoDB.port=8529
arangoDB.user=root
arangoDB.password=111111

24
IP-learning-graph/src/test/java/cn/ac/iie/IpTest.java
View File

@@ -1,10 +1,15 @@
package cn.ac.iie;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseClickhouseData;
import com.arangodb.entity.BaseDocument;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Set;
import java.util.regex.Pattern;
public class IpTest {
@@ -17,7 +22,7 @@ public class IpTest {
System.out.println(s);
System.out.println(pattern.matcher(s).matches());
}
*/
// String ip = "17.57.145.7";
String ip = "pixel.rubiconproject.com";
// String ip = "113.200.17.239";
@@ -25,6 +30,23 @@ public class IpTest {
int hash = Math.abs(ip.hashCode());
int i = hash % ApplicationConfig.THREAD_POOL_NUMBER;
System.out.println(i);
*/
HashMap<Integer, ArrayList<BaseDocument>> vFqdnMap = new HashMap<>();
for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++) {
vFqdnMap.put(i, new ArrayList<>());
}
try {
// ArrayList<BaseDocument> orDefault = vFqdnMap.getOrDefault(-7, new ArrayList<>());
ArrayList<BaseDocument> orDefault = vFqdnMap.get(-7);
System.out.println(orDefault.toString());
Set<Integer> integers = vFqdnMap.keySet();
for (Integer in:integers){
System.out.println(in);
}
}catch (Exception e){
e.printStackTrace();
System.out.println(e.toString());
}
// String[] ipArr = ipStr.split("\\.");
// long ipLong = (Long.valueOf(ipArr[0]) << 24) + (Long.valueOf(ipArr[1]) << 16) + (Long.valueOf(ipArr[2]) << 8) + (Long.valueOf(ipArr[3]));
// System.out.println(ipLong);

45
IP-learning-graph/src/test/java/cn/ac/iie/TestMap.java
View File

@@ -1,17 +1,12 @@
package cn.ac.iie;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.etl.UpdateEFqdnAddressIp;
import cn.ac.iie.dao.BaseArangoData;
import cn.ac.iie.utils.ArangoDBConnect;
import cn.ac.iie.utils.ClickhouseConnect;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.arangodb.ArangoCollection;
import com.arangodb.ArangoDatabase;
import cn.ac.iie.utils.ExecutorThreadPool;
import com.arangodb.entity.*;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
public class TestMap {
@@ -52,6 +47,7 @@ public class TestMap {
*/
String[] distCipRecents = new String[]{"2.3"};
ArrayList<BaseDocument> baseEdgeDocuments = new ArrayList<>();
@@ -77,5 +73,38 @@ public class TestMap {
/*
BaseArangoData.BaseEFqdnAddressIpDataMap();
ExecutorThreadPool.shutdown();
ExecutorThreadPool.awaitThreadTask();
try {
ConcurrentHashMap.KeySetView<String, BaseEdgeDocument> keySet = BaseArangoData.e_Fqdn_Address_Ip_Map.keySet();
ArrayList<BaseEdgeDocument> baseEdgeDocuments = new ArrayList<>();
ArangoDBConnect instance = ArangoDBConnect.getInstance();
for (String key:keySet){
BaseEdgeDocument baseEdgeDocument = BaseArangoData.e_Fqdn_Address_Ip_Map.get(key);
// Long[] tls_cnt_recents = (Long[]) baseEdgeDocument.getAttribute("TLS_CNT_RECENT");
ArrayList<Long> tlsCntRecent = (ArrayList<Long>) baseEdgeDocument.getAttribute("TLS_CNT_RECENT");
Long[] tlsCntRecentsSrc = tlsCntRecent.toArray(new Long[tlsCntRecent.size()]);
System.out.println(Arrays.toString(tlsCntRecentsSrc));
tlsCntRecent.set(tlsCntRecent.size()-2,99L);
baseEdgeDocument.addAttribute("TLS_CNT_RECENT",tlsCntRecent);
baseEdgeDocuments.add(baseEdgeDocument);
}
instance.overwrite(baseEdgeDocuments,"R_LOCATE_FQDN2IP");
}catch (Exception e){
e.printStackTrace();
}finally {
ArangoDBConnect.clean();
}
*/
}
}