From 41e3d4af4ca7824bd88f614afb27ff3e55b95409 Mon Sep 17 00:00:00 2001 From: wangkuan Date: Thu, 26 Sep 2024 16:01:28 +0800 Subject: [PATCH] =?UTF-8?q?09=E7=89=88=E6=9C=AC=E6=94=AF=E6=8C=81=E5=9C=A8?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=A8=A1=E6=9D=BF=E4=B8=AD=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/proxy_event.yaml.j2 | 16 +++++++++----- .../templates/session_record.yaml.j2 | 22 +++++++++---------- .../templates/transaction_record.yaml.j2 | 15 ++++++++----- .../templates/voip_record.yaml.j2 | 15 ++++++++----- .../TSG-24.09/groot_stream/grootstream.yaml | 4 ++++ 5 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 tsg_olap/upgrade/TSG-24.09/groot_stream/grootstream.yaml diff --git a/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 index b57343a..ea38aa7 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 @@ -73,31 +73,32 @@ processing_pipelines: lookup_fields: [rtp_pcap_path] output_fields: [rtp_pcap_path] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path] + path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path] - function: PATH_COMBINE lookup_fields: [http_request_body] output_fields: [http_request_body] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, http_request_body] + path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body] - function: PATH_COMBINE lookup_fields: [http_response_body] output_fields: [http_response_body] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, http_response_body] + path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body] - function: PATH_COMBINE lookup_fields: [mail_eml_file] output_fields: [mail_eml_file] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file] + path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file] - function: PATH_COMBINE lookup_fields: [packet_capture_file] output_fields: [packet_capture_file] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file] + path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file] + - function: CURRENT_UNIX_TIMESTAMP output_fields: [ processing_time ] @@ -143,5 +144,10 @@ application: shade.identifier: aes pipeline: object-reuse: true + properties: + hos.bucket.name.rtp_file: traffic_rtp_file_bucket + hos.bucket.name.http_file: traffic_http_file_bucket + hos.bucket.name.eml_file: traffic_eml_file_bucket + hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket {{ topology }} diff --git a/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 index f3a6237..327bdd3 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 @@ -73,31 +73,31 @@ processing_pipelines: lookup_fields: [rtp_pcap_path] output_fields: [rtp_pcap_path] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path] + path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path] - function: PATH_COMBINE lookup_fields: [http_request_body] output_fields: [http_request_body] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, http_request_body] + path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body] - function: PATH_COMBINE lookup_fields: [http_response_body] output_fields: [http_response_body] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, http_response_body] + path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body] - function: PATH_COMBINE lookup_fields: [mail_eml_file] output_fields: [mail_eml_file] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file] + path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file] - function: PATH_COMBINE lookup_fields: [packet_capture_file] output_fields: [packet_capture_file] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file] + path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file] - function: CURRENT_UNIX_TIMESTAMP output_fields: [ processing_time ] @@ -143,11 +143,9 @@ application: shade.identifier: aes pipeline: object-reuse: true + properties: + hos.bucket.name.rtp_file: traffic_rtp_file_bucket + hos.bucket.name.http_file: traffic_http_file_bucket + hos.bucket.name.eml_file: traffic_eml_file_bucket + hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket {{ topology }} - -# topology: -# - name: kafka_source -# downstream: [etl_processor] -# - name: etl_processor -# downstream: [clickhouse_sink] -# - name: clickhouse_sink diff --git a/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 index 4886509..6bed1a7 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 @@ -73,31 +73,31 @@ processing_pipelines: lookup_fields: [rtp_pcap_path] output_fields: [rtp_pcap_path] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path] + path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path] - function: PATH_COMBINE lookup_fields: [http_request_body] output_fields: [http_request_body] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, http_request_body] + path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body] - function: PATH_COMBINE lookup_fields: [http_response_body] output_fields: [http_response_body] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, http_response_body] + path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body] - function: PATH_COMBINE lookup_fields: [mail_eml_file] output_fields: [mail_eml_file] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file] + path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file] - function: PATH_COMBINE lookup_fields: [packet_capture_file] output_fields: [packet_capture_file] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file] + path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file] - function: CURRENT_UNIX_TIMESTAMP output_fields: [ processing_time ] @@ -143,4 +143,9 @@ application: shade.identifier: aes pipeline: object-reuse: true + properties: + hos.bucket.name.rtp_file: traffic_rtp_file_bucket + hos.bucket.name.http_file: traffic_http_file_bucket + hos.bucket.name.eml_file: traffic_eml_file_bucket + hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket {{ topology }} diff --git a/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 index 64627a5..17fb5b0 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 @@ -73,31 +73,31 @@ processing_pipelines: lookup_fields: [rtp_pcap_path] output_fields: [rtp_pcap_path] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path] + path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path] - function: PATH_COMBINE lookup_fields: [http_request_body] output_fields: [http_request_body] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, http_request_body] + path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body] - function: PATH_COMBINE lookup_fields: [http_response_body] output_fields: [http_response_body] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, http_response_body] + path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body] - function: PATH_COMBINE lookup_fields: [mail_eml_file] output_fields: [mail_eml_file] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file] + path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file] - function: PATH_COMBINE lookup_fields: [packet_capture_file] output_fields: [packet_capture_file] parameters: - path: [props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file] + path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file] - function: CURRENT_UNIX_TIMESTAMP output_fields: [ processing_time ] @@ -143,4 +143,9 @@ application: shade.identifier: aes pipeline: object-reuse: true + properties: + hos.bucket.name.rtp_file: traffic_rtp_file_bucket + hos.bucket.name.http_file: traffic_http_file_bucket + hos.bucket.name.eml_file: traffic_eml_file_bucket + hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket {{ topology }} \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/groot_stream/grootstream.yaml b/tsg_olap/upgrade/TSG-24.09/groot_stream/grootstream.yaml new file mode 100644 index 0000000..0bcbe4d --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/grootstream.yaml @@ -0,0 +1,4 @@ +grootstream: + properties: + hos.path: http://192.168.44.12:9098/hos + scheduler.knowledge_base.update.interval.minutes: 5