gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge from master

Browse Source
This commit is contained in:
wanglihui
2020-08-11 09:30:50 +08:00
parent 9a5cf0126b
commit ad6582893b
2 changed files with 16 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ip-learning-spark/src/main/resources/application.properties
View File

@@ -13,7 +13,7 @@ spark.read.clickhouse.user=default
spark.read.clickhouse.password=111111 spark.read.clickhouse.password=111111
spark.read.clickhouse.numPartitions=144 spark.read.clickhouse.numPartitions=144
spark.read.clickhouse.fetchsize=10000 spark.read.clickhouse.fetchsize=10000
spark.read.clickhouse.partitionColumn=common_recv_time spark.read.clickhouse.partitionColumn=common_start_time
clickhouse.socket.timeout=300000 clickhouse.socket.timeout=300000
#arangoDB配置 #arangoDB配置
arangoDB.host=192.168.40.182 arangoDB.host=192.168.40.182

30
ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala
View File

@@ -31,11 +31,11 @@ object BaseClickhouseData {
} }
def loadConnectionDataFromCk(): Unit ={ def loadConnectionDataFromCk(): Unit ={
val where = "common_recv_time >= " + timeLimit._2 + " AND common_recv_time < " + timeLimit._1 val where = "common_start_time >= " + timeLimit._2 + " AND common_start_time < " + timeLimit._1
val sql = val sql =
s""" s"""
|(SELECT |(SELECT
| ssl_sni,http_host,common_client_ip,common_server_ip,common_recv_time,common_c2s_byte_num,common_s2c_byte_num,common_schema_type | ssl_sni,http_host,common_client_ip,common_server_ip,common_start_time,common_c2s_byte_num,common_s2c_byte_num,common_schema_type
|FROM |FROM
| connection_record_log | connection_record_log
|WHERE $where) as dbtable |WHERE $where) as dbtable
@@ -48,8 +48,8 @@ object BaseClickhouseData {
private def loadRadiusDataFromCk(): Unit ={ private def loadRadiusDataFromCk(): Unit ={
val where = val where =
s""" s"""
| common_recv_time >= ${timeLimit._2} | common_start_time >= ${timeLimit._2}
| AND common_recv_time < ${timeLimit._1} | AND common_start_time < ${timeLimit._1}
| AND common_subscriber_id != '' | AND common_subscriber_id != ''
| AND radius_framed_ip != '' | AND radius_framed_ip != ''
| AND radius_packet_type = 4 | AND radius_packet_type = 4
@@ -58,7 +58,7 @@ object BaseClickhouseData {
val sql = val sql =
s""" s"""
|(SELECT |(SELECT
| common_subscriber_id,radius_framed_ip,common_recv_time | common_subscriber_id,radius_framed_ip,common_start_time
|FROM |FROM
| tsg_galaxy_v3.radius_record_log | tsg_galaxy_v3.radius_record_log
|WHERE |WHERE
@@ -77,7 +77,7 @@ object BaseClickhouseData {
|FROM |FROM
| ( | (
| (SELECT | (SELECT
| ssl_sni AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME | ssl_sni AS FQDN,MAX( common_start_time ) AS LAST_FOUND_TIME,MIN( common_start_time ) AS FIRST_FOUND_TIME
| FROM | FROM
| global_temp.dbtable | global_temp.dbtable
| WHERE | WHERE
@@ -85,7 +85,7 @@ object BaseClickhouseData {
| ) | )
| UNION ALL | UNION ALL
| (SELECT | (SELECT
| http_host AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME | http_host AS FQDN,MAX( common_start_time ) AS LAST_FOUND_TIME,MIN( common_start_time ) AS FIRST_FOUND_TIME
| FROM | FROM
| global_temp.dbtable | global_temp.dbtable
| WHERE | WHERE
@@ -114,8 +114,8 @@ object BaseClickhouseData {
| ( | (
| SELECT | SELECT
| common_client_ip AS IP, | common_client_ip AS IP,
| MIN(common_recv_time) AS FIRST_FOUND_TIME, | MIN(common_start_time) AS FIRST_FOUND_TIME,
| MAX(common_recv_time) AS LAST_FOUND_TIME, | MAX(common_start_time) AS LAST_FOUND_TIME,
| count(*) as SESSION_COUNT, | count(*) as SESSION_COUNT,
| sum(common_c2s_byte_num) as BYTES_SUM, | sum(common_c2s_byte_num) as BYTES_SUM,
| 'client' as ip_type | 'client' as ip_type
@@ -128,8 +128,8 @@ object BaseClickhouseData {
| ( | (
| SELECT | SELECT
| common_server_ip AS IP, | common_server_ip AS IP,
| MIN(common_recv_time) AS FIRST_FOUND_TIME, | MIN(common_start_time) AS FIRST_FOUND_TIME,
| MAX(common_recv_time) AS LAST_FOUND_TIME, | MAX(common_start_time) AS LAST_FOUND_TIME,
| count(*) as SESSION_COUNT, | count(*) as SESSION_COUNT,
| sum(common_s2c_byte_num) as BYTES_SUM, | sum(common_s2c_byte_num) as BYTES_SUM,
| 'server' as ip_type | 'server' as ip_type
@@ -153,8 +153,8 @@ object BaseClickhouseData {
|SELECT |SELECT
| ssl_sni AS FQDN, | ssl_sni AS FQDN,
| common_server_ip, | common_server_ip,
| MAX(common_recv_time) AS LAST_FOUND_TIME, | MAX(common_start_time) AS LAST_FOUND_TIME,
| MIN(common_recv_time) AS FIRST_FOUND_TIME, | MIN(common_start_time) AS FIRST_FOUND_TIME,
| COUNT(*) AS COUNT_TOTAL, | COUNT(*) AS COUNT_TOTAL,
| collect_set(common_client_ip) AS DIST_CIP_RECENT, | collect_set(common_client_ip) AS DIST_CIP_RECENT,
| 'TLS' AS schema_type | 'TLS' AS schema_type
@@ -171,8 +171,8 @@ object BaseClickhouseData {
|SELECT |SELECT
| http_host AS FQDN, | http_host AS FQDN,
| common_server_ip, | common_server_ip,
| MAX(common_recv_time) AS LAST_FOUND_TIME, | MAX(common_start_time) AS LAST_FOUND_TIME,
| MIN(common_recv_time) AS FIRST_FOUND_TIME, | MIN(common_start_time) AS FIRST_FOUND_TIME,
| COUNT(*) AS COUNT_TOTAL, | COUNT(*) AS COUNT_TOTAL,
| collect_set(common_client_ip) AS DIST_CIP_RECENT, | collect_set(common_client_ip) AS DIST_CIP_RECENT,
| 'HTTP' AS schema_type | 'HTTP' AS schema_type