From 3e3678314fb9f06f9c5d37c820bca21533123a55 Mon Sep 17 00:00:00 2001 From: wangkuan Date: Thu, 7 Nov 2024 14:18:43 +0800 Subject: [PATCH] =?UTF-8?q?09=E7=89=88=E6=9C=AC=E5=88=A0=E9=99=A4=E6=A8=A1?= =?UTF-8?q?=E6=9D=BFJSON=20EXTRACT=E5=92=8CDOMAIN=E5=87=BD=E6=95=B0?= =?UTF-8?q?=EF=BC=8C=E7=94=B1=E5=8A=9F=E8=83=BD=E7=AB=AF=E8=A1=A5=E5=85=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/proxy_event.yaml.j2 | 22 +------------------ .../templates/session_record.yaml.j2 | 22 +------------------ .../templates/transaction_record.yaml.j2 | 22 +------------------ .../templates/voip_record.yaml.j2 | 22 +------------------ 4 files changed, 4 insertions(+), 84 deletions(-) diff --git a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/proxy_event.yaml.j2 b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/proxy_event.yaml.j2 index ea38aa7..e793d38 100644 --- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/proxy_event.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/proxy_event.yaml.j2 @@ -18,7 +18,7 @@ sources: processing_pipelines: etl_processor: - type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + type: projection functions: - function: SNOWFLAKE_ID lookup_fields: [''] @@ -26,20 +26,6 @@ processing_pipelines: parameters: data_center_id_num: {{ data_center_id_num }} - - function: JSON_EXTRACT - lookup_fields: [device_tag] - output_fields: [data_center] - filter: - parameters: - value_expression: $.tags[?(@.tag=='data_center')][0].value - - - function: JSON_EXTRACT - lookup_fields: [device_tag] - output_fields: [device_group] - filter: - parameters: - value_expression: $.tags[?(@.tag=='device_group')][0].value - - function: UNIX_TIMESTAMP_CONVERTER lookup_fields: [__timestamp] output_fields: [recv_time] @@ -51,12 +37,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: diff --git a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/session_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/session_record.yaml.j2 index 327bdd3..a1ee302 100644 --- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/session_record.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/session_record.yaml.j2 @@ -18,7 +18,7 @@ sources: processing_pipelines: etl_processor: - type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + type: projection functions: - function: SNOWFLAKE_ID lookup_fields: [''] @@ -26,20 +26,6 @@ processing_pipelines: parameters: data_center_id_num: {{ data_center_id_num }} - - function: JSON_EXTRACT - lookup_fields: [device_tag] - output_fields: [data_center] - filter: - parameters: - value_expression: $.tags[?(@.tag=='data_center')][0].value - - - function: JSON_EXTRACT - lookup_fields: [device_tag] - output_fields: [device_group] - filter: - parameters: - value_expression: $.tags[?(@.tag=='device_group')][0].value - - function: UNIX_TIMESTAMP_CONVERTER lookup_fields: [__timestamp] output_fields: [recv_time] @@ -51,12 +37,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: diff --git a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/transaction_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/transaction_record.yaml.j2 index 6bed1a7..90cbc98 100644 --- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/transaction_record.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/transaction_record.yaml.j2 @@ -18,7 +18,7 @@ sources: processing_pipelines: etl_processor: - type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + type: projection functions: - function: SNOWFLAKE_ID lookup_fields: [''] @@ -26,20 +26,6 @@ processing_pipelines: parameters: data_center_id_num: {{ data_center_id_num }} - - function: JSON_EXTRACT - lookup_fields: [device_tag] - output_fields: [data_center] - filter: - parameters: - value_expression: $.tags[?(@.tag=='data_center')][0].value - - - function: JSON_EXTRACT - lookup_fields: [device_tag] - output_fields: [device_group] - filter: - parameters: - value_expression: $.tags[?(@.tag=='device_group')][0].value - - function: UNIX_TIMESTAMP_CONVERTER lookup_fields: [__timestamp] output_fields: [recv_time] @@ -51,12 +37,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: diff --git a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2 index d36029b..446e4e2 100644 --- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2 @@ -18,7 +18,7 @@ sources: processing_pipelines: etl_processor: - type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl + type: projection functions: - function: SNOWFLAKE_ID lookup_fields: [''] @@ -26,20 +26,6 @@ processing_pipelines: parameters: data_center_id_num: {{ data_center_id_num }} - - function: JSON_EXTRACT - lookup_fields: [device_tag] - output_fields: [data_center] - filter: - parameters: - value_expression: $.tags[?(@.tag=='data_center')][0].value - - - function: JSON_EXTRACT - lookup_fields: [device_tag] - output_fields: [device_group] - filter: - parameters: - value_expression: $.tags[?(@.tag=='device_group')][0].value - - function: UNIX_TIMESTAMP_CONVERTER lookup_fields: [__timestamp] output_fields: [recv_time] @@ -51,12 +37,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: