gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[修复] 修复iplearning查询clickhouse的SQL别名与字段相同引起的查询SSL数据为空问题 (TSG-22783)

Browse Source
This commit is contained in:
zhanghongqing
2024-10-24 11:42:28 +08:00
parent bdf8b00285
commit a43693b2a9
8 changed files with 11 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ip-learning-spark/src/main/scala/cn/ac/iie/config/ApplicationConfig.scala
View File

@@ -14,6 +14,7 @@ object ApplicationConfig {
// val REPARTITION_NUMBER: Int = config.getInt("repartitionNumber")
val MASTER: String = config.getString("master")
val SPARK_SERIALIZER: String = config.getString("spark.serializer")
val SPARK_LOCAL_DIR: String = config.getString("spark.local.dir")
val NUMPARTITIONS: String = config.getString("spark.read.clickhouse.numPartitions")
val SPARK_READ_CLICKHOUSE_URL: String = config.getString("spark.read.clickhouse.url")

4
ip-learning-spark/src/main/scala/cn/ac/iie/dao/BaseClickhouseData.scala
View File

@@ -100,12 +100,12 @@ object BaseClickhouseData {
s"""
|(SELECT * FROM
|((SELECT ssl_sni AS FQDN,server_ip,MAX(recv_time) AS LAST_FOUND_TIME,MIN(recv_time) AS FIRST_FOUND_TIME,COUNT(*) AS COUNT_TOTAL,
|toString(groupUniqArray(${ApplicationConfig.DISTINCT_CLIENT_IP_NUM})(client_ip)) AS DIST_CIP_RECENT,'TLS' AS decoded_as,vsys_id AS VSYS_ID
|toString(groupUniqArray(${ApplicationConfig.DISTINCT_CLIENT_IP_NUM})(client_ip)) AS DIST_CIP_RECENT,'TLS' AS decoded_as_list, vsys_id AS VSYS_ID
|FROM ${ApplicationConfig.SPARK_READ_CLICKHOUSE_SESSION_TABLE}
|WHERE $where and decoded_as = 'SSL' GROUP BY ssl_sni,server_ip,vsys_id)
|UNION ALL
|(SELECT http_host AS FQDN,server_ip,MAX(recv_time) AS LAST_FOUND_TIME,MIN(recv_time) AS FIRST_FOUND_TIME,COUNT(*) AS COUNT_TOTAL,
|toString(groupUniqArray(${ApplicationConfig.DISTINCT_CLIENT_IP_NUM})(client_ip)) AS DIST_CIP_RECENT,'HTTP' AS decoded_as,vsys_id AS VSYS_ID
|toString(groupUniqArray(${ApplicationConfig.DISTINCT_CLIENT_IP_NUM})(client_ip)) AS DIST_CIP_RECENT,'HTTP' AS decoded_as_list,vsys_id AS VSYS_ID
|FROM ${ApplicationConfig.SPARK_READ_CLICKHOUSE_SESSION_TABLE}
|WHERE $where and decoded_as = 'HTTP' GROUP BY http_host,server_ip,vsys_id))
|WHERE FQDN != '') as dbtable

1
ip-learning-spark/src/main/scala/cn/ac/iie/main/IpRecommendApplication.scala
View File

@@ -2,6 +2,7 @@ package cn.ac.iie.main
import cn.ac.iie.service.update.UpdateDocument
@Deprecated
object IpRecommendApplication {
def main(args: Array[String]): Unit = {

1
ip-learning-spark/src/main/scala/cn/ac/iie/main/SubscriberRecommendApplication.scala
View File

@@ -2,6 +2,7 @@ package cn.ac.iie.main
import cn.ac.iie.service.update.UpdateDocument
@Deprecated
object SubscriberRecommendApplication {
def main(args: Array[String]): Unit = {

2
ip-learning-spark/src/main/scala/cn/ac/iie/service/transform/MergeDataFrame.scala
View File

@@ -56,7 +56,7 @@ object MergeDataFrame {
min("FIRST_FOUND_TIME").alias("FIRST_FOUND_TIME"),
max("LAST_FOUND_TIME").alias("LAST_FOUND_TIME"),
collect_list("COUNT_TOTAL").alias("COUNT_TOTAL_LIST"),
collect_list("decoded_as").alias("decoded_as_list"),
collect_list("decoded_as_list").alias("decoded_as_list"),
collect_set("DIST_CIP_RECENT").alias("DIST_CIP_RECENT")
)
val fqdnLocIpRddRow = frame.rdd.map(row => {

1
ip-learning-spark/src/main/scala/cn/ac/iie/utils/SparkSessionUtil.scala
View File

@@ -23,6 +23,7 @@ object SparkSessionUtil {
.config("spark.executor.memory", ApplicationConfig.SPARK_EXECUTOR_MEMORY)
.config("spark.executor.cores",ApplicationConfig.SPARK_EXECUTOR_CORES)
.config("spark.cores.max",ApplicationConfig.SPARK_CORES_MAX)
.config("spark.local.dir",ApplicationConfig.SPARK_LOCAL_DIR)
.config("arangodb.hosts", s"${ApplicationConfig.ARANGODB_HOST}:${ApplicationConfig.ARANGODB_PORT}")
.config("arangodb.user", ApplicationConfig.ARANGODB_USER)
.config("arangodb.password", ApplicationConfig.ARANGODB_PASSWORD)