diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java b/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java index f281b5d..608ab2c 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/dao/UpdateGraphData.java @@ -31,7 +31,7 @@ public class UpdateGraphData { long start = System.currentTimeMillis(); try { updateDocument("ip_learning.r_locate_fqdn2ip_local", LocateFqdn2Ip.class, - ReadClickhouseData::getRelationshipFqdnAddressIpSql,ReadClickhouseData::getRelationFqdnAddressIpDocument); + ReadClickhouseData::getRelationshipFqdnAddressIpSqlYsp,ReadClickhouseData::getRelationFqdnAddressIpDocument); long last = System.currentTimeMillis(); LOG.info("iplearning application运行完毕,用时:"+(last - start)); diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java index 6c0e357..39dd85f 100644 --- a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java +++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java @@ -146,7 +146,8 @@ public class ReadClickhouseData { long firstFoundTime = resultSet.getLong("FIRST_FOUND_TIME"); long lastFoundTime = resultSet.getLong("LAST_FOUND_TIME"); long countTotal = resultSet.getLong("COUNT_TOTAL"); - String schemaType = resultSet.getString("schema_type"); +// String schemaType = resultSet.getString("schema_type"); + String schemaType = ""; String[] distCipRecents = (String[]) resultSet.getArray("DIST_CIP_RECENT").getArray(); long[] clientIpTs = new long[distCipRecents.length]; for (int i = 0; i < clientIpTs.length; i++) { @@ -274,6 +275,11 @@ public class ReadClickhouseData { return "SELECT * FROM ((" + sslSql + ") UNION ALL (" + httpSql + "))WHERE FQDN != ''"; } + public static String getRelationshipFqdnAddressIpSqlYsp() { + String where = "recv_time >= "+minTime+" and recv_time <= "+maxTime +" AND s1_domain != '' AND s1_d_ip != '' "; + return "SELECT s1_domain AS FQDN,s1_d_ip AS common_server_ip,MIN( recv_time ) AS FIRST_FOUND_TIME,MAX( recv_time ) AS LAST_FOUND_TIME,COUNT( * ) AS COUNT_TOTAL,groupUniqArray("+DISTINCT_CLIENT_IP_NUM+")(s1_s_ip) AS DIST_CIP_RECENT FROM media_expire_patch WHERE "+where+" GROUP BY s1_d_ip,s1_domain"; + } + public static String getRelationshipIpVisitFqdnSql() { String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime; String httpSql = "SELECT http_host AS FQDN,common_client_ip,MAX(common_recv_time) AS LAST_FOUND_TIME,MIN(common_recv_time) AS FIRST_FOUND_TIME,COUNT(*) AS COUNT_TOTAL,'HTTP' AS schema_type FROM tsg_galaxy_v3.connection_record_log WHERE " + where + " and common_schema_type = 'HTTP' GROUP BY http_host,common_client_ip"; diff --git a/IP-learning-graph/src/main/resources/application.properties b/IP-learning-graph/src/main/resources/application.properties index 56d8d1c..54e4b57 100644 --- a/IP-learning-graph/src/main/resources/application.properties +++ b/IP-learning-graph/src/main/resources/application.properties @@ -16,8 +16,8 @@ thread.await.termination.time=10 #读取clickhouse时间范围方式,0:读取过去一小时,1:指定时间范围 time.limit.type=1 -read.clickhouse.max.time=1598433621 -read.clickhouse.min.time=1597222501 +read.clickhouse.max.time=1571241610 +read.clickhouse.min.time=1571241600 update.interval=3600 distinct.client.ip.num=10000 diff --git a/IP-learning-graph/src/main/resources/clickhouse.properties b/IP-learning-graph/src/main/resources/clickhouse.properties index 4027174..30a6041 100644 --- a/IP-learning-graph/src/main/resources/clickhouse.properties +++ b/IP-learning-graph/src/main/resources/clickhouse.properties @@ -1,8 +1,10 @@ drivers=ru.yandex.clickhouse.ClickHouseDriver mdb.user=default #db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000 -db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0 -mdb.password=ceiec2019 +#db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0 +#mdb.password=ceiec2019 +db.id=192.168.40.193:8123/tsg_galaxy_zx?socket_timeout=3600000 +mdb.password=111111 initialsize=1 minidle=1 maxactive=50