From 51e0ef2f730554a78842dc55acda15fd55e66975 Mon Sep 17 00:00:00 2001 From: zhanghongqing Date: Thu, 28 Nov 2024 11:55:07 +0800 Subject: [PATCH] =?UTF-8?q?[=E4=BF=AE=E5=A4=8D]=20IP=20Learning=20?= =?UTF-8?q?=E9=80=82=E9=85=8D=20Client/Server=E5=AD=97=E6=AE=B5=E9=87=8D?= =?UTF-8?q?=E5=91=BD=E5=90=8D=E7=9A=84=E5=8F=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scala/cn/ac/iie/dao/BaseClickhouseData.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala b/ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala index 51ac9e6..b3e3c6c 100644 --- a/ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala +++ b/ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala @@ -99,16 +99,16 @@ object BaseClickhouseData { val sql = s""" |(SELECT * FROM - |((SELECT ssl_sni AS FQDN,server_ip AS destination_ip,MAX(recv_time) AS LAST_FOUND_TIME,MIN(recv_time) AS FIRST_FOUND_TIME,COUNT(*) AS COUNT_TOTAL, - |toString(groupUniqArray(${ApplicationConfig.DISTINCT_CLIENT_IP_NUM})(source_ip)) AS DIST_CIP_RECENT,'TLS' AS decoded_as_list, vsys_id AS VSYS_ID + |(SELECT ssl_sni AS FQDN,destination_ip AS server_ip, MAX(recv_time) AS LAST_FOUND_TIME,MIN(recv_time) AS FIRST_FOUND_TIME,COUNT(*) AS COUNT_TOTAL, + |toString(groupUniqArray(${ApplicationConfig.DISTINCT_CLIENT_IP_NUM})(source_ip)) AS DIST_CIP_RECENT, 'TLS' AS decoded_as_list, vsys_id AS VSYS_ID |FROM ${ApplicationConfig.SPARK_READ_CLICKHOUSE_SESSION_TABLE} - |WHERE $where and decoded_as = 'SSL' GROUP BY ssl_sni,server_ip,vsys_id) + |WHERE $where and decoded_as = 'SSL' and notEmpty(ssl_sni) and notEmpty(destination_ip) and vsys_id IS NOT NULL GROUP BY ssl_sni,destination_ip,vsys_id) |UNION ALL - |(SELECT http_host AS FQDN,server_ip AS destination_ip,MAX(recv_time) AS LAST_FOUND_TIME,MIN(recv_time) AS FIRST_FOUND_TIME,COUNT(*) AS COUNT_TOTAL, - |toString(groupUniqArray(${ApplicationConfig.DISTINCT_CLIENT_IP_NUM})(source_ip)) AS DIST_CIP_RECENT,'HTTP' AS decoded_as_list,vsys_id AS VSYS_ID + |(SELECT http_host AS FQDN,destination_ip AS server_ip, MAX(recv_time) AS LAST_FOUND_TIME,MIN(recv_time) AS FIRST_FOUND_TIME,COUNT(*) AS COUNT_TOTAL, + |toString(groupUniqArray(${ApplicationConfig.DISTINCT_CLIENT_IP_NUM})(source_ip)) AS DIST_CIP_RECENT, 'HTTP' AS decoded_as_list,vsys_id AS VSYS_ID |FROM ${ApplicationConfig.SPARK_READ_CLICKHOUSE_SESSION_TABLE} - |WHERE $where and decoded_as = 'HTTP' GROUP BY http_host,server_ip,vsys_id)) - |WHERE FQDN != '') as dbtable + |WHERE $where and decoded_as = 'HTTP' and notEmpty(http_host) and notEmpty(destination_ip) and vsys_id IS NOT NULL GROUP BY http_host,destination_ip,vsys_id) + |) as dbtable """.stripMargin LOG.warn(sql) val frame = initClickhouseData(sql)