From 2a2ad0bf36cab0dcc18ec333018df57cc96c1426 Mon Sep 17 00:00:00 2001 From: zhanghongqing Date: Wed, 19 Jun 2024 13:56:52 +0800 Subject: [PATCH] =?UTF-8?q?GAL-578=20=E6=9B=B4=E6=96=B0groot=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=A8=A1=E6=9D=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/data_transporter.yaml.j2 | 5 ++- .../datapath_telemetry_record.yaml.j2 | 5 ++- ..._kafka_to_clickhouse => dos_event.yaml.j2} | 31 ++++++++++++++----- .../templates/proxy_event.yaml.j2 | 12 ++----- .../templates/session_record.yaml.j2 | 10 ++---- .../templates/traffic_sketch_metric.yaml.j2 | 9 +++--- .../templates/transaction_record.yaml.j2 | 4 +-- .../templates/voip_record.yaml.j2 | 10 ++---- .../templates/data_transporter.yaml.j2 | 5 ++- .../datapath_telemetry_record.yaml.j2 | 5 ++- ..._kafka_to_clickhouse => dos_event.yaml.j2} | 31 ++++++++++++++----- .../templates/proxy_event.yaml.j2 | 12 ++----- .../templates/session_record.yaml.j2 | 10 ++---- .../templates/traffic_sketch_metric.yaml.j2 | 9 +++--- .../templates/transaction_record.yaml.j2 | 4 +-- .../templates/voip_record.yaml.j2 | 10 ++---- 16 files changed, 78 insertions(+), 94 deletions(-) rename tsg_olap/installation/flink/groot_stream/templates/{dos_event_kafka_to_clickhouse => dos_event.yaml.j2} (59%) rename tsg_olap/upgrade/TSG-24.06/groot_stream/templates/{dos_event_kafka_to_clickhouse => dos_event.yaml.j2} (59%) diff --git a/tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2 index e5c6273..5ad1d25 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2 @@ -19,7 +19,7 @@ sinks: type: kafka properties: topic: {{ kafka_sink_topic }} - kafka.bootstrap.servers: { { kafka_sink_servers } } + kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -35,7 +35,7 @@ sinks: clickhouse_sink: type: clickhouse properties: - host: {{ clickhouse_servers }} + host: {{ clickhouse_sink_host }} table: tsg_galaxy_v3.traffic_sketch_metric_local batch.size: 100000 batch.interval: 30s @@ -48,7 +48,6 @@ application: shade.identifier: aes pipeline: object-reuse: true - topology: {{ topology }} diff --git a/tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2 index 58da17c..06058b8 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: DATAPATH-TELEMETRY-RECORD + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -37,7 +37,7 @@ sinks: kafka_sink: type: kafka properties: - topic: DATAPATH-TELEMETRY-RECORD + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 @@ -71,7 +71,6 @@ application: shade.identifier: aes pipeline: object-reuse: true - topology: {{ topology }} diff --git a/tsg_olap/installation/flink/groot_stream/templates/dos_event_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/templates/dos_event.yaml.j2 similarity index 59% rename from tsg_olap/installation/flink/groot_stream/templates/dos_event_kafka_to_clickhouse rename to tsg_olap/installation/flink/groot_stream/templates/dos_event.yaml.j2 index 49a3d1a..e44b182 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/dos_event_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/templates/dos_event.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: DOS-EVENT + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -22,12 +22,30 @@ sources: kafka.compression.type: none format: json - sinks: + kafka_sink: + type: kafka + properties: + topic: {{ kafka_sink_topic }} + kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.retries: 0 + kafka.linger.ms: 10 + kafka.request.timeout.ms: 30000 + kafka.batch.size: 262144 + kafka.buffer.memory: 134217728 + kafka.max.request.size: 10485760 + kafka.compression.type: snappy + kafka.security.protocol: SASL_PLAINTEXT + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + format: json + json.ignore.parse.errors: false + log.failures.only: true + clickhouse_sink: type: clickhouse properties: - host: "{{ clickhouse_servers }}" + host: {{ clickhouse_sink_host }} table: tsg_galaxy_v3.dos_event_local batch.size: 100000 batch.interval: 30s @@ -37,13 +55,10 @@ sinks: application: env: - name: dos_event_kafka_to_clickhouse + name: {{ job_name }} shade.identifier: aes pipeline: object-reuse: true # [boolean] Object Reuse, default is false - topology: - - name: kafka_source - downstream: [clickhouse_sink] - - name: clickhouse_sink + {{ topology }} diff --git a/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 index cee0548..6c750e2 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: PROXY-EVENT + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -50,12 +50,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -107,7 +101,7 @@ sinks: kafka_sink: type: kafka properties: - topic: PROXY-EVENT + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 @@ -141,6 +135,4 @@ application: shade.identifier: aes pipeline: object-reuse: true - topology: {{ topology }} - diff --git a/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 index 8938cd2..7970a95 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: SESSION-RECORD + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -50,12 +50,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -107,7 +101,7 @@ sinks: kafka_sink: type: kafka properties: - topic: SESSION-RECORD + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 diff --git a/tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2 index 0a048a0..571c1af 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: TRAFFIC-SKETCH-METRIC + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -62,12 +62,11 @@ processing_pipelines: parameters: data_center_id_num: 1 - sinks: kafka_sink: type: kafka properties: - topic: TRAFFIC-SKETCH-METRIC + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 @@ -86,7 +85,7 @@ sinks: clickhouse_sink: type: clickhouse properties: - host: {{ clickhouse_servers }} + host: {{ clickhouse_sink_host }} table: tsg_galaxy_v3.traffic_sketch_metric_local batch.size: 100000 batch.interval: 30s @@ -97,7 +96,7 @@ sinks: application: env: # [object] Environment Variables - name: etl_traffic_sketch_metric # [string] Job Name + name: {{ job_name }} shade.identifier: aes pipeline: object-reuse: true # [boolean] Object Reuse, default is false diff --git a/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 index 9c8fa4b..49d1782 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: TRANSACTION-RECORD + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -107,7 +107,7 @@ sinks: kafka_sink: type: kafka properties: - topic: TRANSACTION-RECORD + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 diff --git a/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 index 1a561f2..a37d469 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: VOIP-CONVERSATION-RECORD + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -50,12 +50,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -107,7 +101,7 @@ sinks: kafka_sink: type: kafka properties: - topic: VOIP-CONVERSATION-RECORD + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 diff --git a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/data_transporter.yaml.j2 b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/data_transporter.yaml.j2 index e5c6273..5ad1d25 100644 --- a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/data_transporter.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/data_transporter.yaml.j2 @@ -19,7 +19,7 @@ sinks: type: kafka properties: topic: {{ kafka_sink_topic }} - kafka.bootstrap.servers: { { kafka_sink_servers } } + kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -35,7 +35,7 @@ sinks: clickhouse_sink: type: clickhouse properties: - host: {{ clickhouse_servers }} + host: {{ clickhouse_sink_host }} table: tsg_galaxy_v3.traffic_sketch_metric_local batch.size: 100000 batch.interval: 30s @@ -48,7 +48,6 @@ application: shade.identifier: aes pipeline: object-reuse: true - topology: {{ topology }} diff --git a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/datapath_telemetry_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/datapath_telemetry_record.yaml.j2 index 58da17c..06058b8 100644 --- a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/datapath_telemetry_record.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/datapath_telemetry_record.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: DATAPATH-TELEMETRY-RECORD + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -37,7 +37,7 @@ sinks: kafka_sink: type: kafka properties: - topic: DATAPATH-TELEMETRY-RECORD + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 @@ -71,7 +71,6 @@ application: shade.identifier: aes pipeline: object-reuse: true - topology: {{ topology }} diff --git a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/dos_event_kafka_to_clickhouse b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/dos_event.yaml.j2 similarity index 59% rename from tsg_olap/upgrade/TSG-24.06/groot_stream/templates/dos_event_kafka_to_clickhouse rename to tsg_olap/upgrade/TSG-24.06/groot_stream/templates/dos_event.yaml.j2 index 49a3d1a..e44b182 100644 --- a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/dos_event_kafka_to_clickhouse +++ b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/dos_event.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: DOS-EVENT + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -22,12 +22,30 @@ sources: kafka.compression.type: none format: json - sinks: + kafka_sink: + type: kafka + properties: + topic: {{ kafka_sink_topic }} + kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.retries: 0 + kafka.linger.ms: 10 + kafka.request.timeout.ms: 30000 + kafka.batch.size: 262144 + kafka.buffer.memory: 134217728 + kafka.max.request.size: 10485760 + kafka.compression.type: snappy + kafka.security.protocol: SASL_PLAINTEXT + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + format: json + json.ignore.parse.errors: false + log.failures.only: true + clickhouse_sink: type: clickhouse properties: - host: "{{ clickhouse_servers }}" + host: {{ clickhouse_sink_host }} table: tsg_galaxy_v3.dos_event_local batch.size: 100000 batch.interval: 30s @@ -37,13 +55,10 @@ sinks: application: env: - name: dos_event_kafka_to_clickhouse + name: {{ job_name }} shade.identifier: aes pipeline: object-reuse: true # [boolean] Object Reuse, default is false - topology: - - name: kafka_source - downstream: [clickhouse_sink] - - name: clickhouse_sink + {{ topology }} diff --git a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/proxy_event.yaml.j2 b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/proxy_event.yaml.j2 index cee0548..6c750e2 100644 --- a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/proxy_event.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/proxy_event.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: PROXY-EVENT + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -50,12 +50,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -107,7 +101,7 @@ sinks: kafka_sink: type: kafka properties: - topic: PROXY-EVENT + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 @@ -141,6 +135,4 @@ application: shade.identifier: aes pipeline: object-reuse: true - topology: {{ topology }} - diff --git a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/session_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/session_record.yaml.j2 index 8938cd2..7970a95 100644 --- a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/session_record.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/session_record.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: SESSION-RECORD + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -50,12 +50,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -107,7 +101,7 @@ sinks: kafka_sink: type: kafka properties: - topic: SESSION-RECORD + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 diff --git a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/traffic_sketch_metric.yaml.j2 b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/traffic_sketch_metric.yaml.j2 index 0a048a0..571c1af 100644 --- a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/traffic_sketch_metric.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/traffic_sketch_metric.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: TRAFFIC-SKETCH-METRIC + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -62,12 +62,11 @@ processing_pipelines: parameters: data_center_id_num: 1 - sinks: kafka_sink: type: kafka properties: - topic: TRAFFIC-SKETCH-METRIC + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 @@ -86,7 +85,7 @@ sinks: clickhouse_sink: type: clickhouse properties: - host: {{ clickhouse_servers }} + host: {{ clickhouse_sink_host }} table: tsg_galaxy_v3.traffic_sketch_metric_local batch.size: 100000 batch.interval: 30s @@ -97,7 +96,7 @@ sinks: application: env: # [object] Environment Variables - name: etl_traffic_sketch_metric # [string] Job Name + name: {{ job_name }} shade.identifier: aes pipeline: object-reuse: true # [boolean] Object Reuse, default is false diff --git a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/transaction_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/transaction_record.yaml.j2 index 9c8fa4b..49d1782 100644 --- a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/transaction_record.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/transaction_record.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: TRANSACTION-RECORD + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -107,7 +107,7 @@ sinks: kafka_sink: type: kafka properties: - topic: TRANSACTION-RECORD + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10 diff --git a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/voip_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/voip_record.yaml.j2 index 1a561f2..a37d469 100644 --- a/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/voip_record.yaml.j2 +++ b/tsg_olap/upgrade/TSG-24.06/groot_stream/templates/voip_record.yaml.j2 @@ -2,7 +2,7 @@ sources: kafka_source: type: kafka properties: - topic: VOIP-CONVERSATION-RECORD + topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 @@ -50,12 +50,6 @@ processing_pipelines: parameters: value_expression: recv_time - - function: DOMAIN - lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] - output_fields: [server_domain] - parameters: - option: FIRST_SIGNIFICANT_SUBDOMAIN - - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -107,7 +101,7 @@ sinks: kafka_sink: type: kafka properties: - topic: VOIP-CONVERSATION-RECORD + topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.retries: 0 kafka.linger.ms: 10