gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

过滤所有schema为BASE的原始日志

Browse Source
This commit is contained in:
wanglihui
2020-08-25 14:48:16 +08:00
parent 233cf20d50
commit b7a156b0b8
4 changed files with 9 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java
View File

@@ -265,7 +265,7 @@ public class ReadClickhouseData {
} }
public static String getVertexIpSql() { public static String getVertexIpSql() {
String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime; String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime + " AND common_schema_type != 'BASE'";
String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_c2s) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP"; String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_c2s) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_s2c) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP"; String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_s2c) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
return "SELECT * FROM((" + clientIpSql + ") UNION ALL (" + serverIpSql + "))"; return "SELECT * FROM((" + clientIpSql + ") UNION ALL (" + serverIpSql + "))";

2
IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java
View File

@@ -99,6 +99,8 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
ArrayList<String> distCip = (ArrayList<String>) doc.getAttribute("DIST_CIP"); ArrayList<String> distCip = (ArrayList<String>) doc.getAttribute("DIST_CIP");
ArrayList<Long> distCipTs = (ArrayList<Long>) doc.getAttribute("DIST_CIP_TS"); ArrayList<Long> distCipTs = (ArrayList<Long>) doc.getAttribute("DIST_CIP_TS");
if (distCip == null || distCip.isEmpty()){ if (distCip == null || distCip.isEmpty()){
doc.updateAttribute("DIST_CIP", new String[0]);
doc.updateAttribute("DIST_CIP_TS", new long[0]);
return; return;
} }
distCipTs.add(currentHour - RECENT_COUNT_HOUR * 3600); distCipTs.add(currentHour - RECENT_COUNT_HOUR * 3600);

11
IP-learning-graph/src/main/resources/application.properties
View File

@@ -1,12 +1,9 @@
#arangoDB参数配置 #arangoDB参数配置
arangoDB.host=192.168.40.182 arangoDB.host=192.168.44.12
#arangoDB.host=192.168.40.224
arangoDB.port=8529 arangoDB.port=8529
arangoDB.user=upsert arangoDB.user=upsert
arangoDB.password=ceiec2018 arangoDB.password=ceiec2018
#arangoDB.DB.name=ip-learning-test arangoDB.DB.name=tsg_galaxy_v3
arangoDB.DB.name=ip-learning-test-0
#arangoDB.DB.name=tsg_galaxy_v3
arangoDB.batch=100000 arangoDB.batch=100000
arangoDB.ttl=3600 arangoDB.ttl=3600
@@ -19,8 +16,8 @@ thread.await.termination.time=10
#读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围 #读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围
time.limit.type=0 time.limit.type=0
read.clickhouse.max.time=1596684142 read.clickhouse.max.time=1598323368
read.clickhouse.min.time=1596425769 read.clickhouse.min.time=1597222501
update.interval=3600 update.interval=3600
distinct.client.ip.num=10000 distinct.client.ip.num=10000

7
IP-learning-graph/src/main/resources/clickhouse.properties
View File

@@ -1,10 +1,7 @@
drivers=ru.yandex.clickhouse.ClickHouseDriver drivers=ru.yandex.clickhouse.ClickHouseDriver
mdb.user=default mdb.user=default
#db.id=192.168.40.186:8123/tsg_galaxy_v3?socket_timeout=300000 #db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000
#mdb.password=111111 db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0
db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000&compress=0
#db.id=192.168.40.224:8123/tsg_galaxy_v3?socket_timeout=300000
mdb.password=ceiec2019 mdb.password=ceiec2019
initialsize=1 initialsize=1
minidle=1 minidle=1