Revert "09版本删除JSON EXTRACT和DOMAIN函数，由功能端进行补全"

This reverts commit 179b3d079d.
2024-11-07 14:12:34 +08:00
parent 179b3d079d
commit 4f17e18b5c
5 changed files with 0 additions and 551 deletions
--- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates-all/proxy_event.yaml.j2
+++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates-all/proxy_event.yaml.j2
@@ -1,133 +0,0 @@
-sources:
-  kafka_source:
-    type: kafka
-    properties:
-      topic: PROXY-EVENT
-      kafka.bootstrap.servers: {{ kafka_source_servers }}
-      kafka.client.id: PROXY-EVENT
-      kafka.session.timeout.ms: 60000
-      kafka.max.poll.records: 3000
-      kafka.max.partition.fetch.bytes: 31457280
-      kafka.security.protocol: SASL_PLAINTEXT
-      kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      kafka.group.id: {{ kafka_source_group_id }}
-      kafka.auto.offset.reset: latest
-      format: json
-      json.ignore.parse.errors: false
-
-processing_pipelines:
-  etl_processor:
-    type: projection
-    functions:
-    - function: SNOWFLAKE_ID
-      lookup_fields: ['']
-      output_fields: [log_id]
-      parameters:
-        data_center_id_num: {{ data_center_id_num }}
-
-    - function: UNIX_TIMESTAMP_CONVERTER
-      lookup_fields: [__timestamp]
-      output_fields: [recv_time]
-      parameters:
-        precision: seconds
-
-    - function: EVAL
-      output_fields: [ingestion_time]
-      parameters:
-        value_expression: recv_time
-
-    - function: BASE64_DECODE_TO_STRING
-      output_fields: [mail_subject]
-      parameters:
-        value_field: mail_subject
-        charset_field: mail_subject_charset
-
-    - function: BASE64_DECODE_TO_STRING
-      output_fields: [mail_attachment_name]
-      parameters:
-        value_field: mail_attachment_name
-        charset_field: mail_attachment_name_charset
-
-    - function: PATH_COMBINE
-      lookup_fields: [rtp_pcap_path]
-      output_fields: [rtp_pcap_path]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path]
-
-    - function: PATH_COMBINE
-      lookup_fields: [http_request_body]
-      output_fields: [http_request_body]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body]
-
-    - function: PATH_COMBINE
-      lookup_fields: [http_response_body]
-      output_fields: [http_response_body]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body]
-
-    - function: PATH_COMBINE
-      lookup_fields: [mail_eml_file]
-      output_fields: [mail_eml_file]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file]
-
-    - function: PATH_COMBINE
-      lookup_fields: [packet_capture_file]
-      output_fields: [packet_capture_file]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
-
-
-    - function: CURRENT_UNIX_TIMESTAMP
-      output_fields: [ processing_time ]
-      parameters:
-        precision: seconds
-
-sinks:
-  kafka_sink:
-    type: kafka
-    properties:
-      topic: PROXY-EVENT
-      kafka.bootstrap.servers: {{ kafka_sink_servers }}
-      kafka.client.id: PROXY-EVENT
-      kafka.retries: 0
-      kafka.linger.ms: 10
-      kafka.request.timeout.ms: 30000
-      kafka.batch.size: 262144
-      kafka.buffer.memory: 134217728
-      kafka.max.request.size: 10485760
-      kafka.compression.type: snappy
-      kafka.security.protocol: SASL_PLAINTEXT
-      kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      format: json
-      json.ignore.parse.errors: false
-      log.failures.only: true
-
-  clickhouse_sink:
-    type: clickhouse
-    properties:
-      host: {{ clickhouse_sink_host }}
-      table: tsg_galaxy_v3.proxy_event_local
-      batch.size: 100000
-      batch.interval: 30s
-      connection.user: e54c9568586180eede1506eecf3574e9
-      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
-      connection.connect_timeout: 30
-      connection.query_timeout: 300
-
-application:
-  env: 
-    name: {{ job_name }}
-    shade.identifier: aes
-    pipeline:
-      object-reuse: true
-    properties:
-      hos.bucket.name.rtp_file: traffic_rtp_file_bucket
-      hos.bucket.name.http_file: traffic_http_file_bucket
-      hos.bucket.name.eml_file: traffic_eml_file_bucket
-      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
-  {{ topology }}
-
--- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates-all/session_record.yaml.j2
+++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates-all/session_record.yaml.j2
@@ -1,131 +0,0 @@
-sources:
-  kafka_source:
-    type: kafka
-    properties:
-      topic: SESSION-RECORD
-      kafka.bootstrap.servers: {{ kafka_source_servers }}
-      kafka.client.id: SESSION-RECORD
-      kafka.session.timeout.ms: 60000
-      kafka.max.poll.records: 3000
-      kafka.max.partition.fetch.bytes: 31457280
-      kafka.security.protocol: SASL_PLAINTEXT
-      kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      kafka.group.id: {{ kafka_source_group_id }}
-      kafka.auto.offset.reset: latest
-      format: json
-      json.ignore.parse.errors: false
-
-processing_pipelines:
-  etl_processor:
-    type: projection
-    functions:
-    - function: SNOWFLAKE_ID
-      lookup_fields: ['']
-      output_fields: [log_id]
-      parameters:
-        data_center_id_num: {{ data_center_id_num }}
-
-    - function: UNIX_TIMESTAMP_CONVERTER
-      lookup_fields: [__timestamp]
-      output_fields: [recv_time]
-      parameters:
-        precision: seconds
-
-    - function: EVAL
-      output_fields: [ingestion_time]
-      parameters:
-        value_expression: recv_time
-
-    - function: BASE64_DECODE_TO_STRING
-      output_fields: [mail_subject]
-      parameters:
-        value_field: mail_subject
-        charset_field: mail_subject_charset
-
-    - function: BASE64_DECODE_TO_STRING
-      output_fields: [mail_attachment_name]
-      parameters:
-        value_field: mail_attachment_name
-        charset_field: mail_attachment_name_charset
-
-    - function: PATH_COMBINE
-      lookup_fields: [rtp_pcap_path]
-      output_fields: [rtp_pcap_path]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path]
-
-    - function: PATH_COMBINE
-      lookup_fields: [http_request_body]
-      output_fields: [http_request_body]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body]
-
-    - function: PATH_COMBINE
-      lookup_fields: [http_response_body]
-      output_fields: [http_response_body]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body]
-
-    - function: PATH_COMBINE
-      lookup_fields: [mail_eml_file]
-      output_fields: [mail_eml_file]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file]
-
-    - function: PATH_COMBINE
-      lookup_fields: [packet_capture_file]
-      output_fields: [packet_capture_file]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
-
-    - function: CURRENT_UNIX_TIMESTAMP
-      output_fields: [ processing_time ]
-      parameters:
-        precision: seconds
-
-sinks:
-  kafka_sink:
-    type: kafka
-    properties:
-      topic: SESSION-RECORD
-      kafka.bootstrap.servers: {{ kafka_sink_servers }}
-      kafka.client.id: SESSION-RECORD
-      kafka.retries: 0
-      kafka.linger.ms: 10
-      kafka.request.timeout.ms: 30000
-      kafka.batch.size: 262144
-      kafka.buffer.memory: 134217728
-      kafka.max.request.size: 10485760
-      kafka.compression.type: snappy
-      kafka.security.protocol: SASL_PLAINTEXT
-      kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      format: json
-      json.ignore.parse.errors: false
-      log.failures.only: true
-
-  clickhouse_sink:
-    type: clickhouse
-    properties:
-      host: {{ clickhouse_sink_host }}
-      table: tsg_galaxy_v3.session_record_local
-      batch.size: 100000
-      batch.interval: 30s
-      connection.user: e54c9568586180eede1506eecf3574e9
-      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
-      connection.connect_timeout: 30
-      connection.query_timeout: 300
-
-application:
-  env:
-    name: {{ job_name }}
-    shade.identifier: aes
-    pipeline:
-      object-reuse: true
-    properties:
-      hos.bucket.name.rtp_file: traffic_rtp_file_bucket
-      hos.bucket.name.http_file: traffic_http_file_bucket
-      hos.bucket.name.eml_file: traffic_eml_file_bucket
-      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
-  {{ topology }}
--- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates-all/transaction_record.yaml.j2
+++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates-all/transaction_record.yaml.j2
@@ -1,131 +0,0 @@
-sources:
-  kafka_source:
-    type: kafka
-    properties:
-      topic: TRANSACTION-RECORD
-      kafka.bootstrap.servers: {{ kafka_source_servers }}
-      kafka.client.id: TRANSACTION-RECORD
-      kafka.session.timeout.ms: 60000
-      kafka.max.poll.records: 3000
-      kafka.max.partition.fetch.bytes: 31457280
-      kafka.security.protocol: SASL_PLAINTEXT
-      kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      kafka.group.id: {{ kafka_source_group_id }}
-      kafka.auto.offset.reset: latest
-      format: json
-      json.ignore.parse.errors: false
-
-processing_pipelines:
-  etl_processor:
-    type: projection
-    functions:
-    - function: SNOWFLAKE_ID
-      lookup_fields: ['']
-      output_fields: [log_id]
-      parameters:
-        data_center_id_num: {{ data_center_id_num }}
-
-    - function: UNIX_TIMESTAMP_CONVERTER
-      lookup_fields: [__timestamp]
-      output_fields: [recv_time]
-      parameters:
-        precision: seconds
-
-    - function: EVAL
-      output_fields: [ingestion_time]
-      parameters:
-        value_expression: recv_time
-
-    - function: BASE64_DECODE_TO_STRING
-      output_fields: [mail_subject]
-      parameters:
-        value_field: mail_subject
-        charset_field: mail_subject_charset
-
-    - function: BASE64_DECODE_TO_STRING
-      output_fields: [mail_attachment_name]
-      parameters:
-        value_field: mail_attachment_name
-        charset_field: mail_attachment_name_charset
-
-    - function: PATH_COMBINE
-      lookup_fields: [rtp_pcap_path]
-      output_fields: [rtp_pcap_path]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path]
-
-    - function: PATH_COMBINE
-      lookup_fields: [http_request_body]
-      output_fields: [http_request_body]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body]
-
-    - function: PATH_COMBINE
-      lookup_fields: [http_response_body]
-      output_fields: [http_response_body]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body]
-
-    - function: PATH_COMBINE
-      lookup_fields: [mail_eml_file]
-      output_fields: [mail_eml_file]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file]
-
-    - function: PATH_COMBINE
-      lookup_fields: [packet_capture_file]
-      output_fields: [packet_capture_file]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
-
-    - function: CURRENT_UNIX_TIMESTAMP
-      output_fields: [ processing_time ]
-      parameters:
-        precision: seconds
-
-sinks:
-  kafka_sink:
-    type: kafka
-    properties:
-      topic: TRANSACTION-RECORD
-      kafka.bootstrap.servers: {{ kafka_sink_servers }}
-      kafka.client.id: TRANSACTION-RECORD
-      kafka.retries: 0
-      kafka.linger.ms: 10
-      kafka.request.timeout.ms: 30000
-      kafka.batch.size: 262144
-      kafka.buffer.memory: 134217728
-      kafka.max.request.size: 10485760
-      kafka.compression.type: snappy
-      kafka.security.protocol: SASL_PLAINTEXT
-      kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      format: json
-      json.ignore.parse.errors: false
-      log.failures.only: true
-
-  clickhouse_sink:
-    type: clickhouse
-    properties:
-      host: {{ clickhouse_sink_host }}
-      table: tsg_galaxy_v3.transaction_record_local
-      batch.size: 100000
-      batch.interval: 30s
-      connection.user: e54c9568586180eede1506eecf3574e9
-      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
-      connection.connect_timeout: 30
-      connection.query_timeout: 300
-
-application:
-  env: 
-    name: {{ job_name }}
-    shade.identifier: aes
-    pipeline:
-      object-reuse: true
-    properties:
-      hos.bucket.name.rtp_file: traffic_rtp_file_bucket
-      hos.bucket.name.http_file: traffic_http_file_bucket
-      hos.bucket.name.eml_file: traffic_eml_file_bucket
-      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
-  {{ topology }}
--- a/tsg_olap/upgrade/TSG-24.09/groot_stream/templates-all/voip_record.yaml.j2
+++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/templates-all/voip_record.yaml.j2
@@ -1,132 +0,0 @@
-sources:
-  kafka_source:
-    type: kafka
-    properties:
-      topic: VOIP-CONVERSATION-RECORD
-      kafka.bootstrap.servers: {{ kafka_source_servers }}
-      kafka.client.id: VOIP-CONVERSATION-RECORD
-      kafka.session.timeout.ms: 60000
-      kafka.max.poll.records: 3000
-      kafka.max.partition.fetch.bytes: 31457280
-      kafka.security.protocol: SASL_PLAINTEXT
-      kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      kafka.group.id: {{ kafka_source_group_id }}
-      kafka.auto.offset.reset: latest
-      format: json
-      json.ignore.parse.errors: false
-
-processing_pipelines:
-  etl_processor:
-    type: projection
-    functions:
-    - function: SNOWFLAKE_ID
-      lookup_fields: ['']
-      output_fields: [log_id]
-      parameters:
-        data_center_id_num: {{ data_center_id_num }}
-
-    - function: UNIX_TIMESTAMP_CONVERTER
-      lookup_fields: [__timestamp]
-      output_fields: [recv_time]
-      parameters:
-        precision: seconds
-
-    - function: EVAL
-      output_fields: [ingestion_time]
-      parameters:
-        value_expression: recv_time
-
-
-    - function: BASE64_DECODE_TO_STRING
-      output_fields: [mail_subject]
-      parameters:
-        value_field: mail_subject
-        charset_field: mail_subject_charset
-
-    - function: BASE64_DECODE_TO_STRING
-      output_fields: [mail_attachment_name]
-      parameters:
-        value_field: mail_attachment_name
-        charset_field: mail_attachment_name_charset
-
-    - function: PATH_COMBINE
-      lookup_fields: [rtp_pcap_path]
-      output_fields: [rtp_pcap_path]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path]
-
-    - function: PATH_COMBINE
-      lookup_fields: [http_request_body]
-      output_fields: [http_request_body]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body]
-
-    - function: PATH_COMBINE
-      lookup_fields: [http_response_body]
-      output_fields: [http_response_body]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body]
-
-    - function: PATH_COMBINE
-      lookup_fields: [mail_eml_file]
-      output_fields: [mail_eml_file]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file]
-
-    - function: PATH_COMBINE
-      lookup_fields: [packet_capture_file]
-      output_fields: [packet_capture_file]
-      parameters:
-        path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
-
-    - function: CURRENT_UNIX_TIMESTAMP
-      output_fields: [ processing_time ]
-      parameters:
-        precision: seconds
-
-sinks:
-  kafka_sink:
-    type: kafka
-    properties:
-      topic: VOIP-CONVERSATION-RECORD
-      kafka.bootstrap.servers: {{ kafka_sink_servers }}
-      kafka.client.id: VOIP-CONVERSATION-RECORD
-      kafka.retries: 0
-      kafka.linger.ms: 10
-      kafka.request.timeout.ms: 30000
-      kafka.batch.size: 262144
-      kafka.buffer.memory: 134217728
-      kafka.max.request.size: 10485760
-      kafka.compression.type: snappy
-      kafka.security.protocol: SASL_PLAINTEXT
-      kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
-      format: json
-      json.ignore.parse.errors: false
-      log.failures.only: true
-
-  clickhouse_sink:
-    type: clickhouse
-    properties:
-      host: {{ clickhouse_sink_host }}
-      table: tsg_galaxy_v3.voip_record_local
-      batch.size: 100000
-      batch.interval: 30s
-      connection.user: e54c9568586180eede1506eecf3574e9
-      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
-      connection.connect_timeout: 30
-      connection.query_timeout: 300
-
-application:
-  env: 
-    name: {{ job_name }}
-    shade.identifier: aes
-    pipeline:
-      object-reuse: true
-    properties:
-      hos.bucket.name.rtp_file: traffic_rtp_file_bucket
-      hos.bucket.name.http_file: traffic_http_file_bucket
-      hos.bucket.name.eml_file: traffic_eml_file_bucket
-      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
-  {{ topology }}
--- a/tsg_olap/upgrade/TSG-24.09/groot_stream/udf.plugins
+++ b/tsg_olap/upgrade/TSG-24.09/groot_stream/udf.plugins
@@ -1,24 +0,0 @@
-com.geedgenetworks.core.udf.AsnLookup
-com.geedgenetworks.core.udf.CurrentUnixTimestamp
-com.geedgenetworks.core.udf.DecodeBase64
-com.geedgenetworks.core.udf.Domain
-com.geedgenetworks.core.udf.Drop
-com.geedgenetworks.core.udf.EncodeBase64
-com.geedgenetworks.core.udf.Eval
-com.geedgenetworks.core.udf.Flatten
-com.geedgenetworks.core.udf.FromUnixTimestamp
-com.geedgenetworks.core.udf.GenerateStringArray
-com.geedgenetworks.core.udf.GeoIpLookup
-com.geedgenetworks.core.udf.JsonExtract
-com.geedgenetworks.core.udf.PathCombine
-com.geedgenetworks.core.udf.Rename
-com.geedgenetworks.core.udf.SnowflakeId
-com.geedgenetworks.core.udf.StringJoiner
-com.geedgenetworks.core.udf.UnixTimestampConverter
-com.geedgenetworks.core.udf.udaf.NumberSum
-com.geedgenetworks.core.udf.udaf.CollectList
-com.geedgenetworks.core.udf.udaf.CollectSet
-com.geedgenetworks.core.udf.udaf.LongCount
-com.geedgenetworks.core.udf.udaf.Mean
-com.geedgenetworks.core.udf.udaf.LastValue
-com.geedgenetworks.core.udf.udaf.FirstValue