gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge from master

Browse Source
This commit is contained in:
wanglihui
2020-08-11 09:30:50 +08:00
parent 9a5cf0126b
commit ad6582893b
2 changed files with 16 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ip-learning-spark/src/main/resources/application.properties
View File

@@ -13,7 +13,7 @@ spark.read.clickhouse.user=default
spark.read.clickhouse.password=111111
spark.read.clickhouse.numPartitions=144
spark.read.clickhouse.fetchsize=10000
spark.read.clickhouse.partitionColumn=common_recv_time
spark.read.clickhouse.partitionColumn=common_start_time
clickhouse.socket.timeout=300000
#arangoDB配置
arangoDB.host=192.168.40.182

30
ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala
View File

@@ -31,11 +31,11 @@ object BaseClickhouseData {
}
def loadConnectionDataFromCk(): Unit ={
val where = "common_recv_time >= " + timeLimit._2 + " AND common_recv_time < " + timeLimit._1
val where = "common_start_time >= " + timeLimit._2 + " AND common_start_time < " + timeLimit._1
val sql =
s"""
|(SELECT
| ssl_sni,http_host,common_client_ip,common_server_ip,common_recv_time,common_c2s_byte_num,common_s2c_byte_num,common_schema_type
| ssl_sni,http_host,common_client_ip,common_server_ip,common_start_time,common_c2s_byte_num,common_s2c_byte_num,common_schema_type
|FROM
| connection_record_log
|WHERE $where) as dbtable
@@ -48,8 +48,8 @@ object BaseClickhouseData {
private def loadRadiusDataFromCk(): Unit ={
val where =
s"""
| common_recv_time >= ${timeLimit._2}
| AND common_recv_time < ${timeLimit._1}
| common_start_time >= ${timeLimit._2}
| AND common_start_time < ${timeLimit._1}
| AND common_subscriber_id != ''
| AND radius_framed_ip != ''
| AND radius_packet_type = 4
@@ -58,7 +58,7 @@ object BaseClickhouseData {
val sql =
s"""
|(SELECT
| common_subscriber_id,radius_framed_ip,common_recv_time
| common_subscriber_id,radius_framed_ip,common_start_time
|FROM
| tsg_galaxy_v3.radius_record_log
|WHERE
@@ -77,7 +77,7 @@ object BaseClickhouseData {
|FROM
| (
| (SELECT
| ssl_sni AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME
| ssl_sni AS FQDN,MAX( common_start_time ) AS LAST_FOUND_TIME,MIN( common_start_time ) AS FIRST_FOUND_TIME
| FROM
| global_temp.dbtable
| WHERE
@@ -85,7 +85,7 @@ object BaseClickhouseData {
| )
| UNION ALL
| (SELECT
| http_host AS FQDN,MAX( common_recv_time ) AS LAST_FOUND_TIME,MIN( common_recv_time ) AS FIRST_FOUND_TIME
| http_host AS FQDN,MAX( common_start_time ) AS LAST_FOUND_TIME,MIN( common_start_time ) AS FIRST_FOUND_TIME
| FROM
| global_temp.dbtable
| WHERE
@@ -114,8 +114,8 @@ object BaseClickhouseData {
| (
| SELECT
| common_client_ip AS IP,
| MIN(common_recv_time) AS FIRST_FOUND_TIME,
| MAX(common_recv_time) AS LAST_FOUND_TIME,
| MIN(common_start_time) AS FIRST_FOUND_TIME,
| MAX(common_start_time) AS LAST_FOUND_TIME,
| count(*) as SESSION_COUNT,
| sum(common_c2s_byte_num) as BYTES_SUM,
| 'client' as ip_type
@@ -128,8 +128,8 @@ object BaseClickhouseData {
| (
| SELECT
| common_server_ip AS IP,
| MIN(common_recv_time) AS FIRST_FOUND_TIME,
| MAX(common_recv_time) AS LAST_FOUND_TIME,
| MIN(common_start_time) AS FIRST_FOUND_TIME,
| MAX(common_start_time) AS LAST_FOUND_TIME,
| count(*) as SESSION_COUNT,
| sum(common_s2c_byte_num) as BYTES_SUM,
| 'server' as ip_type
@@ -153,8 +153,8 @@ object BaseClickhouseData {
|SELECT
| ssl_sni AS FQDN,
| common_server_ip,
| MAX(common_recv_time) AS LAST_FOUND_TIME,
| MIN(common_recv_time) AS FIRST_FOUND_TIME,
| MAX(common_start_time) AS LAST_FOUND_TIME,
| MIN(common_start_time) AS FIRST_FOUND_TIME,
| COUNT(*) AS COUNT_TOTAL,
| collect_set(common_client_ip) AS DIST_CIP_RECENT,
| 'TLS' AS schema_type
@@ -171,8 +171,8 @@ object BaseClickhouseData {
|SELECT
| http_host AS FQDN,
| common_server_ip,
| MAX(common_recv_time) AS LAST_FOUND_TIME,
| MIN(common_recv_time) AS FIRST_FOUND_TIME,
| MAX(common_start_time) AS LAST_FOUND_TIME,
| MIN(common_start_time) AS FIRST_FOUND_TIME,
| COUNT(*) AS COUNT_TOTAL,
| collect_set(common_client_ip) AS DIST_CIP_RECENT,
| 'HTTP' AS schema_type