From 1552ed0a6c5a1800c44c3edb0b0e2b5e9ec13ccb Mon Sep 17 00:00:00 2001
From: wangkuan <wangkuan@geedgenetworks.com>
Date: Thu, 7 Nov 2024 14:38:35 +0800
Subject: [PATCH] =?UTF-8?q?GAL-682=20BASE64=5FENCODE=5FTO=5FSTRING?=
 =?UTF-8?q?=E5=87=BD=E6=95=B0=E6=96=B0=E5=A2=9E=E5=8F=82=E6=95=B0input=5Ft?=
 =?UTF-8?q?ype,=E6=94=AF=E6=8C=81string,byte=5Farray=E7=B1=BB=E5=9E=8B?=
 =?UTF-8?q?=EF=BC=8C=E4=BB=BB=E5=8A=A1=E6=A8=A1=E6=9D=BF=E9=80=82=E9=85=8D?=
 =?UTF-8?q?=E5=8F=98=E6=9B=B4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../datapath_telemetry_record.yaml.j2         | 80 +++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 tsg_olap/upgrade/TSG-24.10/groot_stream/templates/datapath_telemetry_record.yaml.j2

diff --git a/tsg_olap/upgrade/TSG-24.10/groot_stream/templates/datapath_telemetry_record.yaml.j2 b/tsg_olap/upgrade/TSG-24.10/groot_stream/templates/datapath_telemetry_record.yaml.j2
new file mode 100644
index 0000000..10554d6
--- /dev/null
+++ b/tsg_olap/upgrade/TSG-24.10/groot_stream/templates/datapath_telemetry_record.yaml.j2
@@ -0,0 +1,80 @@
+sources:
+  kafka_source:
+    type: kafka
+    properties:
+      topic: DATAPATH-TELEMETRY-RECORD
+      kafka.bootstrap.servers: {{ kafka_source_servers }}
+      kafka.client.id: DATAPATH-TELEMETRY-RECORD
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      kafka.group.id: {{ kafka_source_group_id }}
+      kafka.auto.offset.reset: latest
+      format: msgpack
+
+processing_pipelines:
+  etl_processor:
+    type: projection
+    functions:
+      - function: SNOWFLAKE_ID
+        lookup_fields: [ '' ]
+        output_fields: [ log_id ]
+        parameters:
+          data_center_id_num: {{ data_center_id_num }}
+      - function: UNIX_TIMESTAMP_CONVERTER
+        lookup_fields: [ __timestamp ]
+        output_fields: [ recv_time ]
+        parameters:
+          precision: seconds
+      - function: BASE64_ENCODE_TO_STRING
+        lookup_fields: [packet]
+        output_fields: [packet]
+        parameters:
+          input_type: byte_array
+
+
+
+sinks:
+  kafka_sink:
+    type: kafka
+    properties:
+      topic: DATAPATH-TELEMETRY-RECORD
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.client.id: DATAPATH-TELEMETRY-RECORD
+      kafka.retries: 0
+      kafka.linger.ms: 10
+      kafka.request.timeout.ms: 30000
+      kafka.batch.size: 262144
+      kafka.buffer.memory: 134217728
+      kafka.max.request.size: 10485760
+      kafka.compression.type: snappy
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      format: raw
+      json.ignore.parse.errors: false
+      log.failures.only: true
+
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_sink_host }}
+      table: tsg_galaxy_v3.datapath_telemetry_record_local
+      batch.size: 5000
+      batch.interval: 30s
+      connection.user: e54c9568586180eede1506eecf3574e9
+      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
+      connection.connect_timeout: 30
+      connection.query_timeout: 300
+
+application:
+  env:
+    name: {{ job_name }}
+    shade.identifier: aes
+    pipeline:
+      object-reuse: true
+  {{ topology }}
+