GAL-578 更新groot任务模板

tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2

@@ -19,7 +19,7 @@ sinks:
     type: kafka
     properties:
       topic: {{ kafka_sink_topic }}
-      kafka.bootstrap.servers: { { kafka_sink_servers } }
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10
       kafka.request.timeout.ms: 30000
@@ -35,7 +35,7 @@ sinks:
   clickhouse_sink:
     type: clickhouse
     properties:
-      host: {{ clickhouse_servers }}
+      host: {{ clickhouse_sink_host }}
       table: tsg_galaxy_v3.traffic_sketch_metric_local
       batch.size: 100000
       batch.interval: 30s
@@ -48,7 +48,6 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
 
 

tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: DATAPATH-TELEMETRY-RECORD
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -37,7 +37,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: DATAPATH-TELEMETRY-RECORD
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10
@@ -71,7 +71,6 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
 
 

tsg_olap/installation/flink/groot_stream/templates/dos_event.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: DOS-EVENT
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: "{{ kafka_source_servers }}"
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -22,12 +22,30 @@ sources:
       kafka.compression.type: none
       format: json
 
-
 sinks:
+  kafka_sink:
+    type: kafka
+    properties:
+      topic: {{ kafka_sink_topic }}
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.retries: 0
+      kafka.linger.ms: 10
+      kafka.request.timeout.ms: 30000
+      kafka.batch.size: 262144
+      kafka.buffer.memory: 134217728
+      kafka.max.request.size: 10485760
+      kafka.compression.type: snappy
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
+      format: json
+      json.ignore.parse.errors: false
+      log.failures.only: true
+
   clickhouse_sink:
     type: clickhouse
     properties:
-      host: "{{ clickhouse_servers }}"
+      host: {{ clickhouse_sink_host }}
       table: tsg_galaxy_v3.dos_event_local
       batch.size: 100000
       batch.interval: 30s
@@ -37,13 +55,10 @@ sinks:
 
 application:
   env:
-    name: dos_event_kafka_to_clickhouse
+    name: {{ job_name }}
     shade.identifier: aes
     pipeline:
       object-reuse: true # [boolean] Object Reuse, default is false
-  topology:
-  - name: kafka_source
-    downstream: [clickhouse_sink]
-  - name: clickhouse_sink
+  {{ topology }}
 
 

tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: PROXY-EVENT
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -50,12 +50,6 @@ processing_pipelines:
       parameters:
         value_expression: recv_time
 
-    - function: DOMAIN
-      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
-      output_fields: [server_domain]
-      parameters:
-        option: FIRST_SIGNIFICANT_SUBDOMAIN
-
     - function: BASE64_DECODE_TO_STRING
       output_fields: [mail_subject]
       parameters:
@@ -107,7 +101,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: PROXY-EVENT
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10
@@ -141,6 +135,4 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
-

tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: SESSION-RECORD
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -50,12 +50,6 @@ processing_pipelines:
       parameters:
         value_expression: recv_time
 
-    - function: DOMAIN
-      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
-      output_fields: [server_domain]
-      parameters:
-        option: FIRST_SIGNIFICANT_SUBDOMAIN
-
     - function: BASE64_DECODE_TO_STRING
       output_fields: [mail_subject]
       parameters:
@@ -107,7 +101,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: SESSION-RECORD
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10

tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: TRAFFIC-SKETCH-METRIC
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -62,12 +62,11 @@ processing_pipelines:
         parameters:
           data_center_id_num: 1
 
-
 sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: TRAFFIC-SKETCH-METRIC
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10
@@ -86,7 +85,7 @@ sinks:
   clickhouse_sink:
     type: clickhouse
     properties:
-      host: {{ clickhouse_servers }}
+      host: {{ clickhouse_sink_host }}
       table: tsg_galaxy_v3.traffic_sketch_metric_local
       batch.size: 100000
       batch.interval: 30s
@@ -97,7 +96,7 @@ sinks:
 application:
 
   env: # [object] Environment Variables
-    name: etl_traffic_sketch_metric # [string] Job Name
+    name: {{ job_name }}
     shade.identifier: aes
     pipeline:
       object-reuse: true # [boolean] Object Reuse, default is false

tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: TRANSACTION-RECORD
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -107,7 +107,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: TRANSACTION-RECORD
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10

tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: VOIP-CONVERSATION-RECORD
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -50,12 +50,6 @@ processing_pipelines:
       parameters:
         value_expression: recv_time
 
-    - function: DOMAIN
-      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
-      output_fields: [server_domain]
-      parameters:
-        option: FIRST_SIGNIFICANT_SUBDOMAIN
-
     - function: BASE64_DECODE_TO_STRING
       output_fields: [mail_subject]
       parameters:
@@ -107,7 +101,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: VOIP-CONVERSATION-RECORD
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10

tsg_olap/upgrade/TSG-24.06/groot_stream/templates/data_transporter.yaml.j2

@@ -19,7 +19,7 @@ sinks:
     type: kafka
     properties:
       topic: {{ kafka_sink_topic }}
-      kafka.bootstrap.servers: { { kafka_sink_servers } }
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10
       kafka.request.timeout.ms: 30000
@@ -35,7 +35,7 @@ sinks:
   clickhouse_sink:
     type: clickhouse
     properties:
-      host: {{ clickhouse_servers }}
+      host: {{ clickhouse_sink_host }}
       table: tsg_galaxy_v3.traffic_sketch_metric_local
       batch.size: 100000
       batch.interval: 30s
@@ -48,7 +48,6 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
 
 

tsg_olap/upgrade/TSG-24.06/groot_stream/templates/datapath_telemetry_record.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: DATAPATH-TELEMETRY-RECORD
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -37,7 +37,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: DATAPATH-TELEMETRY-RECORD
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10
@@ -71,7 +71,6 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
 
 

tsg_olap/upgrade/TSG-24.06/groot_stream/templates/dos_event.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: DOS-EVENT
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: "{{ kafka_source_servers }}"
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -22,12 +22,30 @@ sources:
       kafka.compression.type: none
       format: json
 
-
 sinks:
+  kafka_sink:
+    type: kafka
+    properties:
+      topic: {{ kafka_sink_topic }}
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.retries: 0
+      kafka.linger.ms: 10
+      kafka.request.timeout.ms: 30000
+      kafka.batch.size: 262144
+      kafka.buffer.memory: 134217728
+      kafka.max.request.size: 10485760
+      kafka.compression.type: snappy
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
+      format: json
+      json.ignore.parse.errors: false
+      log.failures.only: true
+
   clickhouse_sink:
     type: clickhouse
     properties:
-      host: "{{ clickhouse_servers }}"
+      host: {{ clickhouse_sink_host }}
       table: tsg_galaxy_v3.dos_event_local
       batch.size: 100000
       batch.interval: 30s
@@ -37,13 +55,10 @@ sinks:
 
 application:
   env:
-    name: dos_event_kafka_to_clickhouse
+    name: {{ job_name }}
     shade.identifier: aes
     pipeline:
       object-reuse: true # [boolean] Object Reuse, default is false
-  topology:
-  - name: kafka_source
-    downstream: [clickhouse_sink]
-  - name: clickhouse_sink
+  {{ topology }}
 
 

tsg_olap/upgrade/TSG-24.06/groot_stream/templates/proxy_event.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: PROXY-EVENT
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -50,12 +50,6 @@ processing_pipelines:
       parameters:
         value_expression: recv_time
 
-    - function: DOMAIN
-      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
-      output_fields: [server_domain]
-      parameters:
-        option: FIRST_SIGNIFICANT_SUBDOMAIN
-
     - function: BASE64_DECODE_TO_STRING
       output_fields: [mail_subject]
       parameters:
@@ -107,7 +101,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: PROXY-EVENT
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10
@@ -141,6 +135,4 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
-

tsg_olap/upgrade/TSG-24.06/groot_stream/templates/session_record.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: SESSION-RECORD
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -50,12 +50,6 @@ processing_pipelines:
       parameters:
         value_expression: recv_time
 
-    - function: DOMAIN
-      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
-      output_fields: [server_domain]
-      parameters:
-        option: FIRST_SIGNIFICANT_SUBDOMAIN
-
     - function: BASE64_DECODE_TO_STRING
       output_fields: [mail_subject]
       parameters:
@@ -107,7 +101,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: SESSION-RECORD
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10

tsg_olap/upgrade/TSG-24.06/groot_stream/templates/traffic_sketch_metric.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: TRAFFIC-SKETCH-METRIC
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -62,12 +62,11 @@ processing_pipelines:
         parameters:
           data_center_id_num: 1
 
-
 sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: TRAFFIC-SKETCH-METRIC
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10
@@ -86,7 +85,7 @@ sinks:
   clickhouse_sink:
     type: clickhouse
     properties:
-      host: {{ clickhouse_servers }}
+      host: {{ clickhouse_sink_host }}
       table: tsg_galaxy_v3.traffic_sketch_metric_local
       batch.size: 100000
       batch.interval: 30s
@@ -97,7 +96,7 @@ sinks:
 application:
 
   env: # [object] Environment Variables
-    name: etl_traffic_sketch_metric # [string] Job Name
+    name: {{ job_name }}
     shade.identifier: aes
     pipeline:
       object-reuse: true # [boolean] Object Reuse, default is false

tsg_olap/upgrade/TSG-24.06/groot_stream/templates/transaction_record.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: TRANSACTION-RECORD
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -107,7 +107,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: TRANSACTION-RECORD
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10

tsg_olap/upgrade/TSG-24.06/groot_stream/templates/voip_record.yaml.j2

@@ -2,7 +2,7 @@ sources:
   kafka_source:
     type: kafka
     properties:
-      topic: VOIP-CONVERSATION-RECORD
+      topic: {{ kafka_source_topic }}
       kafka.bootstrap.servers: {{ kafka_source_servers }}
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
@@ -50,12 +50,6 @@ processing_pipelines:
       parameters:
         value_expression: recv_time
 
-    - function: DOMAIN
-      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
-      output_fields: [server_domain]
-      parameters:
-        option: FIRST_SIGNIFICANT_SUBDOMAIN
-
     - function: BASE64_DECODE_TO_STRING
       output_fields: [mail_subject]
       parameters:
@@ -107,7 +101,7 @@ sinks:
   kafka_sink:
     type: kafka
     properties:
-      topic: VOIP-CONVERSATION-RECORD
+      topic: {{ kafka_sink_topic }}
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
       kafka.retries: 0
       kafka.linger.ms: 10