gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

过滤所有schema为BASE的原始日志

Browse Source
This commit is contained in:
wanglihui
2020-08-25 14:48:16 +08:00
parent 233cf20d50
commit b7a156b0b8
4 changed files with 9 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java
View File

@@ -265,7 +265,7 @@ public class ReadClickhouseData {
}
public static String getVertexIpSql() {
String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime;
String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime + " AND common_schema_type != 'BASE'";
String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_c2s) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_s2c) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
return "SELECT * FROM((" + clientIpSql + ") UNION ALL (" + serverIpSql + "))";

2
IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java
View File

@@ -99,6 +99,8 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
ArrayList<String> distCip = (ArrayList<String>) doc.getAttribute("DIST_CIP");
ArrayList<Long> distCipTs = (ArrayList<Long>) doc.getAttribute("DIST_CIP_TS");
if (distCip == null || distCip.isEmpty()){
doc.updateAttribute("DIST_CIP", new String[0]);
doc.updateAttribute("DIST_CIP_TS", new long[0]);
return;
}
distCipTs.add(currentHour - RECENT_COUNT_HOUR * 3600);

11
IP-learning-graph/src/main/resources/application.properties
View File

@@ -1,12 +1,9 @@
#arangoDB参数配置
arangoDB.host=192.168.40.182
#arangoDB.host=192.168.40.224
arangoDB.host=192.168.44.12
arangoDB.port=8529
arangoDB.user=upsert
arangoDB.password=ceiec2018
#arangoDB.DB.name=ip-learning-test
arangoDB.DB.name=ip-learning-test-0
#arangoDB.DB.name=tsg_galaxy_v3
arangoDB.DB.name=tsg_galaxy_v3
arangoDB.batch=100000
arangoDB.ttl=3600
@@ -19,8 +16,8 @@ thread.await.termination.time=10
#读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围
time.limit.type=0
read.clickhouse.max.time=1596684142
read.clickhouse.min.time=1596425769
read.clickhouse.max.time=1598323368
read.clickhouse.min.time=1597222501
update.interval=3600
distinct.client.ip.num=10000

7
IP-learning-graph/src/main/resources/clickhouse.properties
View File

@@ -1,10 +1,7 @@
drivers=ru.yandex.clickhouse.ClickHouseDriver
mdb.user=default
#db.id=192.168.40.186:8123/tsg_galaxy_v3?socket_timeout=300000
#mdb.password=111111
db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000&compress=0
#db.id=192.168.40.224:8123/tsg_galaxy_v3?socket_timeout=300000
#db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000
db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0
mdb.password=ceiec2019
initialsize=1
minidle=1