gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

切换YSP数据

Browse Source
This commit is contained in:
wanglihui
2020-08-27 17:44:58 +08:00
parent 17ec1eff72
commit 1017d8fe6c
4 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java
View File

@@ -31,7 +31,7 @@ public class UpdateGraphData {
long start = System.currentTimeMillis();
try {
updateDocument("ip_learning.r_locate_fqdn2ip_local", LocateFqdn2Ip.class,
ReadClickhouseData::getRelationshipFqdnAddressIpSql,ReadClickhouseData::getRelationFqdnAddressIpDocument);
ReadClickhouseData::getRelationshipFqdnAddressIpSqlYsp,ReadClickhouseData::getRelationFqdnAddressIpDocument);
long last = System.currentTimeMillis();
LOG.info("iplearning application运行完毕，用时："+(last - start));

8
IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java
View File

@@ -146,7 +146,8 @@ public class ReadClickhouseData {
long firstFoundTime = resultSet.getLong("FIRST_FOUND_TIME");
long lastFoundTime = resultSet.getLong("LAST_FOUND_TIME");
long countTotal = resultSet.getLong("COUNT_TOTAL");
String schemaType = resultSet.getString("schema_type");
// String schemaType = resultSet.getString("schema_type");
String schemaType = "";
String[] distCipRecents = (String[]) resultSet.getArray("DIST_CIP_RECENT").getArray();
long[] clientIpTs = new long[distCipRecents.length];
for (int i = 0; i < clientIpTs.length; i++) {
@@ -274,6 +275,11 @@ public class ReadClickhouseData {
return "SELECT * FROM ((" + sslSql + ") UNION ALL (" + httpSql + "))WHERE FQDN != ''";
}
public static String getRelationshipFqdnAddressIpSqlYsp() {
String where = "recv_time >= "+minTime+" and recv_time <= "+maxTime +" AND s1_domain != '' AND s1_d_ip != '' ";
return "SELECT s1_domain AS FQDN,s1_d_ip AS common_server_ip,MIN( recv_time ) AS FIRST_FOUND_TIME,MAX( recv_time ) AS LAST_FOUND_TIME,COUNT( * ) AS COUNT_TOTAL,groupUniqArray("+DISTINCT_CLIENT_IP_NUM+")(s1_s_ip) AS DIST_CIP_RECENT FROM media_expire_patch WHERE "+where+" GROUP BY s1_d_ip,s1_domain";
}
public static String getRelationshipIpVisitFqdnSql() {
String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime;
String httpSql = "SELECT http_host AS FQDN,common_client_ip,MAX(common_recv_time) AS LAST_FOUND_TIME,MIN(common_recv_time) AS FIRST_FOUND_TIME,COUNT(*) AS COUNT_TOTAL,'HTTP' AS schema_type FROM tsg_galaxy_v3.connection_record_log WHERE " + where + " and common_schema_type = 'HTTP' GROUP BY http_host,common_client_ip";

4
IP-learning-graph/src/main/resources/application.properties
View File

@@ -16,8 +16,8 @@ thread.await.termination.time=10
#读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围
time.limit.type=1
read.clickhouse.max.time=1598433621
read.clickhouse.min.time=1597222501
read.clickhouse.max.time=1571241610
read.clickhouse.min.time=1571241600
update.interval=3600
distinct.client.ip.num=10000

6
IP-learning-graph/src/main/resources/clickhouse.properties
View File

@@ -1,8 +1,10 @@
drivers=ru.yandex.clickhouse.ClickHouseDriver
mdb.user=default
#db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000
db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0
mdb.password=ceiec2019
#db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0
#mdb.password=ceiec2019
db.id=192.168.40.193:8123/tsg_galaxy_zx?socket_timeout=3600000
mdb.password=111111
initialsize=1
minidle=1
maxactive=50