diff --git a/ip-learning-spark/src/main/resources/application.properties b/ip-learning-spark/src/main/resources/application.properties index 0010b23..2ccecef 100644 --- a/ip-learning-spark/src/main/resources/application.properties +++ b/ip-learning-spark/src/main/resources/application.properties @@ -13,7 +13,7 @@ spark.read.clickhouse.user=default spark.read.clickhouse.password=111111 spark.read.clickhouse.numPartitions=144 spark.read.clickhouse.fetchsize=10000 -spark.read.clickhouse.partitionColumn=common_recv_time +spark.read.clickhouse.partitionColumn=common_start_time clickhouse.socket.timeout=300000 #arangoDB配置 arangoDB.host=192.168.40.182 diff --git a/ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala b/ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala index 952c30c..c0d6c6b 100644 --- a/ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala +++ b/ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala @@ -31,11 +31,11 @@ object BaseClickhouseData { } def loadConnectionDataFromCk(): Unit ={ - val where = "common_recv_time >= " + timeLimit._2 + " AND common_recv_time < " + timeLimit._1 + val where = "common_start_time >= " + timeLimit._2 + " AND common_start_time < " + timeLimit._1 val sql = s""" |(SELECT - | ssl_sni,http_host,common_client_ip,common_server_ip,common_recv_time,common_c2s_byte_num,common_s2c_byte_num,common_schema_type + | ssl_sni,http_host,common_client_ip,common_server_ip,common_start_time,common_c2s_byte_num,common_s2c_byte_num,common_schema_type |FROM | connection_record_log |WHERE $where) as dbtable @@ -48,8 +48,8 @@ object BaseClickhouseData { private def loadRadiusDataFromCk(): Unit ={ val where = s""" - | common_recv_time >= ${timeLimit._2} - | AND common_recv_time < ${timeLimit._1} + | common_start_time >= ${timeLimit._2} + | AND common_start_time < ${timeLimit._1} | AND common_subscriber_id != '' | AND radius_framed_ip != '' | AND radius_packet_type = 4 @@ -58,7 +58,7 @@ object BaseClickhouseData { val sql = s""" |(SELECT - | common_subscriber_id,radius_framed_ip,common_recv_time + | common_subscriber_id,radius_framed_ip,common_start_time |FROM | tsg_galaxy_v3.radius_record_log |WHERE @@ -77,7 +77,7 @@ object BaseClickhouseData { |FROM | ( | (SELECT - | ssl_sni AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME + | ssl_sni AS FQDN,MAX( common_start_time ) AS LAST_FOUND_TIME,MIN( common_start_time ) AS FIRST_FOUND_TIME | FROM | global_temp.dbtable | WHERE @@ -85,7 +85,7 @@ object BaseClickhouseData { | ) | UNION ALL | (SELECT - | http_host AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME + | http_host AS FQDN,MAX( common_start_time ) AS LAST_FOUND_TIME,MIN( common_start_time ) AS FIRST_FOUND_TIME | FROM | global_temp.dbtable | WHERE @@ -114,8 +114,8 @@ object BaseClickhouseData { | ( | SELECT | common_client_ip AS IP, - | MIN(common_recv_time) AS FIRST_FOUND_TIME, - | MAX(common_recv_time) AS LAST_FOUND_TIME, + | MIN(common_start_time) AS FIRST_FOUND_TIME, + | MAX(common_start_time) AS LAST_FOUND_TIME, | count(*) as SESSION_COUNT, | sum(common_c2s_byte_num) as BYTES_SUM, | 'client' as ip_type @@ -128,8 +128,8 @@ object BaseClickhouseData { | ( | SELECT | common_server_ip AS IP, - | MIN(common_recv_time) AS FIRST_FOUND_TIME, - | MAX(common_recv_time) AS LAST_FOUND_TIME, + | MIN(common_start_time) AS FIRST_FOUND_TIME, + | MAX(common_start_time) AS LAST_FOUND_TIME, | count(*) as SESSION_COUNT, | sum(common_s2c_byte_num) as BYTES_SUM, | 'server' as ip_type @@ -153,8 +153,8 @@ object BaseClickhouseData { |SELECT | ssl_sni AS FQDN, | common_server_ip, - | MAX(common_recv_time) AS LAST_FOUND_TIME, - | MIN(common_recv_time) AS FIRST_FOUND_TIME, + | MAX(common_start_time) AS LAST_FOUND_TIME, + | MIN(common_start_time) AS FIRST_FOUND_TIME, | COUNT(*) AS COUNT_TOTAL, | collect_set(common_client_ip) AS DIST_CIP_RECENT, | 'TLS' AS schema_type @@ -171,8 +171,8 @@ object BaseClickhouseData { |SELECT | http_host AS FQDN, | common_server_ip, - | MAX(common_recv_time) AS LAST_FOUND_TIME, - | MIN(common_recv_time) AS FIRST_FOUND_TIME, + | MAX(common_start_time) AS LAST_FOUND_TIME, + | MIN(common_start_time) AS FIRST_FOUND_TIME, | COUNT(*) AS COUNT_TOTAL, | collect_set(common_client_ip) AS DIST_CIP_RECENT, | 'HTTP' AS schema_type