gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Flink任务中Kafka Source/Sink增加client.id，修改kafka.sasl.jaas.config用户为olap（TSG-21676）

Browse Source
This commit is contained in:
wangchengcheng
2024-08-15 16:01:10 +08:00
parent ee318e993a
commit ac113caf04
35 changed files with 167 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
tsg_olap/installation/flink/groot_stream/README.md
View File

@@ -1,27 +1,3 @@
# 敏感配置信息加密
AES encryption support encrypt/decrypt the following parameters:
- username
- password
- auth
- connection.user
- connection.password
- kafka.sasl.jaas.config
- kafka.ssl.keystore.password
- kafka.ssl.truststore.password
- kafka.ssl.key.password
## Encrypt/Decrypt 举例
- 使用Encrypt命令对配置文件的明文敏感信息转换为密文
`${GROOTSTREAM_HOME}/bin/start.sh -c config/xx.yaml --encrypt `
- 使用Decrypt命令，将配置加密的信息转换为明文
`${GROOTSTREAM_HOME}/bin/start.sh -c config/xx.yaml --decrypt `
# 配置模版举例 # 配置模版举例
## session_record.yaml.j2 （会话日志ETL场景） ## session_record.yaml.j2 （会话日志ETL场景）

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/dos_sketch_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: DOS-SKETCH-RECORD topic: DOS-SKETCH-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: DOS-SKETCH-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: dos_sketch_record_kafka_to_kafka kafka.group.id: dos_sketch_record_kafka_to_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: raw format: raw
@@ -17,10 +18,11 @@ sources:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: DOS-SKETCH-RECORD topic: DOS-SKETCH-RECORD
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: DOS-SKETCH-RECORD
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -30,7 +32,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_proxy_event_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: PROXY-EVENT topic: PROXY-EVENT
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: PROXY-EVENT
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: etl_proxy_event_kafka_to_ndc_kafka kafka.group.id: etl_proxy_event_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -118,10 +119,11 @@ processing_pipelines:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: PROXY-EVENT-PROCESSED topic: PROXY-EVENT-PROCESSED
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: PROXY-EVENT-PROCESSED
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -131,7 +133,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_session_record_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: SESSION-RECORD topic: SESSION-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: SESSION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: etl_session_record_kafka_to_ndc_kafka kafka.group.id: etl_session_record_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -117,10 +118,11 @@ processing_pipelines:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: SESSION-RECORD-PROCESSED topic: SESSION-RECORD-PROCESSED
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: SESSION-RECORD-PROCESSED
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -130,7 +132,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_transaction_record_kafka_to_ndc_kafka
View File

@@ -8,12 +8,13 @@ sources:
properties: properties:
topic: TRANSACTION-RECORD topic: TRANSACTION-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: TRANSACTION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: etl_transaction_record_kafka_to_ndc_kafka kafka.group.id: etl_transaction_record_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -121,10 +122,11 @@ processing_pipelines:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: TRANSACTION-RECORD-PROCESSED topic: TRANSACTION-RECORD-PROCESSED
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: TRANSACTION-RECORD-PROCESSED
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -134,7 +136,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/network_traffic_metrics_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: NETWORK-TRAFFIC-METRIC topic: NETWORK-TRAFFIC-METRIC
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: NETWORK-TRAFFIC-METRIC
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: network_traffic_metrics_kafka_to_ndc_kafka kafka.group.id: network_traffic_metrics_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: raw format: raw
@@ -17,10 +18,11 @@ sources:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: NETWORK-TRAFFIC-METRIC topic: NETWORK-TRAFFIC-METRIC
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: NETWORK-TRAFFIC-METRIC
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -30,7 +32,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw

9
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/object_statistics_metric_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: OBJECT-STATISTICS-METRIC topic: OBJECT-STATISTICS-METRIC
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: OBJECT-STATISTICS-METRIC
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: object_statistics_metric_kafka_to_ndc_kafka kafka.group.id: object_statistics_metric_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: raw format: raw
@@ -17,9 +18,11 @@ sources:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: OBJECT-STATISTICS-METRIC
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: OBJECT-STATISTICS-METRIC
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -29,7 +32,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/policy_rule_metrics_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: POLICY-RULE-METRIC topic: POLICY-RULE-METRIC
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: POLICY-RULE-METRIC
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: policy_rule_metrics_kafka_to_ndc_kafka kafka.group.id: policy_rule_metrics_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: raw format: raw
@@ -17,10 +18,11 @@ sources:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: POLICY-RULE-METRIC topic: POLICY-RULE-METRIC
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: POLICY-RULE-METRIC
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -30,7 +32,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/pxy_exch_intermedia_cert_kafka_to_ndc_kafka
View File

@@ -4,6 +4,7 @@ sources:
properties: properties:
topic: PXY-EXCH-INTERMEDIA-CERT topic: PXY-EXCH-INTERMEDIA-CERT
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: PXY-EXCH-INTERMEDIA-CERT
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
@@ -14,7 +15,7 @@ sources:
kafka.ssl.truststore.password: kafka.ssl.truststore.password:
kafka.ssl.key.password: kafka.ssl.key.password:
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.buffer.memory: kafka.buffer.memory:
kafka.group.id: pxy_exch_intermedia_cert_kafka_to_ndc_kafka kafka.group.id: pxy_exch_intermedia_cert_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
@@ -25,10 +26,11 @@ sources:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: PXY-EXCH-INTERMEDIA-CERT topic: PXY-EXCH-INTERMEDIA-CERT
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: PXY-EXCH-INTERMEDIA-CERT
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -38,7 +40,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/statistics_rule_metric_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: STATISTICS-RULE-METRIC topic: STATISTICS-RULE-METRIC
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: STATISTICS-RULE-METRIC
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: statistics_rule_metric_kafka_to_ndc_kafka kafka.group.id: statistics_rule_metric_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: raw format: raw
@@ -17,10 +18,11 @@ sources:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: STATISTICS-RULE-METRIC topic: STATISTICS-RULE-METRIC
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: STATISTICS-RULE-METRIC
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -30,7 +32,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/troubleshooting_file_stream_record_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: TROUBLESHOOTING-FILE-STREAM-RECORD topic: TROUBLESHOOTING-FILE-STREAM-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: TROUBLESHOOTING-FILE-STREAM-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: troubleshooting_file_stream_record_kafka_to_ndc_kafka kafka.group.id: troubleshooting_file_stream_record_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: raw format: raw
@@ -17,10 +18,11 @@ sources:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: TROUBLESHOOTING-FILE-STREAM-RECORD topic: TROUBLESHOOTING-FILE-STREAM-RECORD
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: TROUBLESHOOTING-FILE-STREAM-RECORD
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -30,7 +32,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw

8
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/voip_record_kafka_to_ndc_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: VOIP-RECORD topic: VOIP-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: VOIP-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: statistics_rule_metric_kafka_to_ndc_kafka kafka.group.id: statistics_rule_metric_kafka_to_ndc_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -55,10 +56,11 @@ processing_pipelines:
sinks: sinks:
kafka_sink: kafka_sink:
type : kafka type: kafka
properties: properties:
topic: VOIP-RECORD topic: VOIP-RECORD
kafka.bootstrap.servers: "{{ kafka_sink_servers }}" kafka.bootstrap.servers: "{{ kafka_sink_servers }}"
kafka.client.id: VOIP-RECORD
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -68,7 +70,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json

3
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/dos_event_kafka_to_clickhouse
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: DOS-EVENT topic: DOS-EVENT
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: DOS-EVENT
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: dos_event_kafka_to_clickhouse kafka.group.id: dos_event_kafka_to_clickhouse
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json

6
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_session_record_processed_kafka_to_cn_kafka
View File

@@ -4,12 +4,13 @@ sources:
properties: # [object] Source Properties properties: # [object] Source Properties
topic: SESSION-RECORD-PROCESSED topic: SESSION-RECORD-PROCESSED
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: SESSION-RECORD-PROCESSED
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: etl_processed_session_record_kafka_to_cn_kafka kafka.group.id: etl_processed_session_record_kafka_to_cn_kafka
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -362,6 +363,7 @@ sinks:
properties: properties:
topic: SESSION-RECORD-CN topic: SESSION-RECORD-CN
kafka.bootstrap.servers: {{ national_center_cn_kafka_servers }} kafka.bootstrap.servers: {{ national_center_cn_kafka_servers }}
kafka.client.id: SESSION-RECORD-CN
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -371,7 +373,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json

3
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_voip_record_kafka_to_clickhouse
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: VOIP-CONVERSATION-RECORD topic: VOIP-CONVERSATION-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: VOIP-CONVERSATION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: etl_voip_record_kafka_to_clickhouse kafka.group.id: etl_voip_record_kafka_to_clickhouse
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json

3
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/proxy_event_processed_kafka_to_clickhouse
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: PROXY-EVENT-PROCESSED topic: PROXY-EVENT-PROCESSED
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: PROXY-EVENT-PROCESSED
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: proxy_event_processed_kafka_to_clickhouse kafka.group.id: proxy_event_processed_kafka_to_clickhouse
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json

3
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/session_record_processed_kafka_to_clickhouse
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: SESSION-RECORD-PROCESSED topic: SESSION-RECORD-PROCESSED
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: SESSION-RECORD-PROCESSED
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: session_record_processed_kafka_to_clickhouse kafka.group.id: session_record_processed_kafka_to_clickhouse
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json

3
tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/transaction_record_processed_kafka_to_clickhouse
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: TRANSACTION-RECORD-PROCESSED topic: TRANSACTION-RECORD-PROCESSED
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: TRANSACTION-RECORD-PROCESSED
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: transaction_record_processed_kafka_to_clickhouse kafka.group.id: transaction_record_processed_kafka_to_clickhouse
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json

6
tsg_olap/installation/flink/groot_stream/realtime_log_streaming_cn_session_record.yaml.template
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: {{ kafka_source_topic }} topic: {{ kafka_source_topic }}
kafka.bootstrap.servers: {{ kafka_source_bootstrap_servers }} kafka.bootstrap.servers: {{ kafka_source_bootstrap_servers }}
kafka.client.id: {{ kafka_source_topic }}
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: {{ kafka_source_group_id }} kafka.group.id: {{ kafka_source_group_id }}
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -354,6 +355,7 @@ sinks:
properties: properties:
topic: {{ kafka_sink_topic }} topic: {{ kafka_sink_topic }}
kafka.bootstrap.servers: {{ kafka_sink_bootstrap_servers }} kafka.bootstrap.servers: {{ kafka_sink_bootstrap_servers }}
kafka.client.id: {{ kafka_sink_topic }}
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -363,7 +365,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json
json.ignore.parse.errors: false json.ignore.parse.errors: false
log.failures.only: true log.failures.only: true

3
tsg_olap/installation/flink/groot_stream/single-cluster-examples/dos_event_kafka_to_clickhouse
View File

@@ -4,6 +4,7 @@ sources:
properties: properties:
topic: DOS-EVENT topic: DOS-EVENT
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: DOS-EVENT
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
@@ -14,7 +15,7 @@ sources:
kafka.ssl.truststore.password: kafka.ssl.truststore.password:
kafka.ssl.key.password: kafka.ssl.key.password:
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.buffer.memory: kafka.buffer.memory:
kafka.group.id: dos_event_kafka_to_clickhouse-20231221 kafka.group.id: dos_event_kafka_to_clickhouse-20231221
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest

3
tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_datapath_telemetry_record_kafka_to_clickhouse
View File

@@ -4,6 +4,7 @@ sources:
properties: properties:
topic: DATAPATH-TELEMETRY-RECORD topic: DATAPATH-TELEMETRY-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: DATAPATH-TELEMETRY-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
@@ -16,7 +17,7 @@ sources:
#kafka.ssl.key.password: 86cf0e2ffba3f541a6c6761313e5cc7e #kafka.ssl.key.password: 86cf0e2ffba3f541a6c6761313e5cc7e
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: etl_datapath_telemetry_record_kafka_to_clickhouse-20230125 kafka.group.id: etl_datapath_telemetry_record_kafka_to_clickhouse-20230125
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: msgpack format: msgpack

3
tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_proxy_event_kafka_to_clickhouse
View File

@@ -8,6 +8,7 @@ sources:
properties: properties:
topic: PROXY-EVENT topic: PROXY-EVENT
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: PROXY-EVENT
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
@@ -18,7 +19,7 @@ sources:
kafka.ssl.truststore.password: kafka.ssl.truststore.password:
kafka.ssl.key.password: kafka.ssl.key.password:
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.buffer.memory: kafka.buffer.memory:
kafka.group.id: etl_proxy_event_kafka_to_clickhouse-20231221 kafka.group.id: etl_proxy_event_kafka_to_clickhouse-20231221
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest

3
tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_session_record_kafka_to_clickhouse
View File

@@ -8,6 +8,7 @@ sources:
properties: properties:
topic: SESSION-RECORD topic: SESSION-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: SESSION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
@@ -18,7 +19,7 @@ sources:
kafka.ssl.truststore.password: kafka.ssl.truststore.password:
kafka.ssl.key.password: kafka.ssl.key.password:
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.buffer.memory: kafka.buffer.memory:
kafka.group.id: etl_session_record_kafka_to_clickhouse-20230125 kafka.group.id: etl_session_record_kafka_to_clickhouse-20230125
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest

3
tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_traffic_sketch_metric_kafka_to_clickhouse
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: TRAFFIC-SKETCH-METRIC topic: TRAFFIC-SKETCH-METRIC
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: TRAFFIC-SKETCH-METRIC
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: etl_traffic_sketch_metric kafka.group.id: etl_traffic_sketch_metric
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
kafka.compression.type: none kafka.compression.type: none

3
tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_transaction_record_kafka_to_clickhouse
View File

@@ -8,6 +8,7 @@ sources:
properties: properties:
topic: TRANSACTION-RECORD topic: TRANSACTION-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: TRANSACTION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
@@ -18,7 +19,7 @@ sources:
kafka.ssl.truststore.password: kafka.ssl.truststore.password:
kafka.ssl.key.password: kafka.ssl.key.password:
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.buffer.memory: kafka.buffer.memory:
kafka.group.id: etl_transaction_record_kafka_to_clickhouse-20240308 kafka.group.id: etl_transaction_record_kafka_to_clickhouse-20240308
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest

3
tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_voip_record_kafka_to_clickhouse
View File

@@ -8,6 +8,7 @@ sources:
properties: properties:
topic: VOIP-CONVERSATION-RECORD topic: VOIP-CONVERSATION-RECORD
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: VOIP-CONVERSATION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
@@ -18,7 +19,7 @@ sources:
kafka.ssl.truststore.password: kafka.ssl.truststore.password:
kafka.ssl.key.password: kafka.ssl.key.password:
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.buffer.memory: kafka.buffer.memory:
kafka.group.id: etl_voip_record_kafka_to_clickhouse-20231221 kafka.group.id: etl_voip_record_kafka_to_clickhouse-20231221
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest

11
tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2
View File

@@ -4,12 +4,13 @@ sources:
properties: properties:
topic: {{ kafka_source_topic }} topic: {{ kafka_source_topic }}
kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.client.id: {{ kafka_source_topic }}
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: {{ kafka_source_group_id }} kafka.group.id: {{ kafka_source_group_id }}
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: raw format: raw
@@ -19,7 +20,8 @@ sinks:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_sink_topic }} topic: {{ kafka_sink_topic }}
kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.bootstrap.servers: { { kafka_sink_servers } }
kafka.client.id: {{ kafka_sink_topic }}
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -29,13 +31,13 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw
clickhouse_sink: clickhouse_sink:
type: clickhouse type: clickhouse
properties: properties:
host: {{ clickhouse_sink_host }} host: {{ clickhouse_servers }}
table: tsg_galaxy_v3.traffic_sketch_metric_local table: tsg_galaxy_v3.traffic_sketch_metric_local
batch.size: 100000 batch.size: 100000
batch.interval: 30s batch.interval: 30s
@@ -48,6 +50,7 @@ application:
shade.identifier: aes shade.identifier: aes
pipeline: pipeline:
object-reuse: true object-reuse: true
topology:
{{ topology }} {{ topology }}

11
tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2
View File

@@ -2,14 +2,15 @@ sources:
kafka_source: kafka_source:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_source_topic }} topic: DATAPATH-TELEMETRY-RECORD
kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.client.id: DATAPATH-TELEMETRY-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: {{ kafka_source_group_id }} kafka.group.id: {{ kafka_source_group_id }}
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: msgpack format: msgpack
@@ -37,8 +38,9 @@ sinks:
kafka_sink: kafka_sink:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_sink_topic }} topic: DATAPATH-TELEMETRY-RECORD
kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.client.id: DATAPATH-TELEMETRY-RECORD
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -48,7 +50,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: raw format: raw
json.ignore.parse.errors: false json.ignore.parse.errors: false
log.failures.only: true log.failures.only: true
@@ -71,6 +73,7 @@ application:
shade.identifier: aes shade.identifier: aes
pipeline: pipeline:
object-reuse: true object-reuse: true
topology:
{{ topology }} {{ topology }}

34
tsg_olap/installation/flink/groot_stream/templates/dos_event.yaml.j2 → tsg_olap/installation/flink/groot_stream/templates/dos_event_kafka_to_clickhouse
View File

@@ -2,8 +2,9 @@ sources:
kafka_source: kafka_source:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_source_topic }} topic: DOS-EVENT
kafka.bootstrap.servers: "{{ kafka_source_servers }}" kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.client.id: DOS-EVENT
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
@@ -14,7 +15,7 @@ sources:
kafka.ssl.truststore.password: kafka.ssl.truststore.password:
kafka.ssl.key.password: kafka.ssl.key.password:
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.buffer.memory: kafka.buffer.memory:
kafka.group.id: dos_event_kafka_to_clickhouse-20231221 kafka.group.id: dos_event_kafka_to_clickhouse-20231221
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
@@ -22,30 +23,12 @@ sources:
kafka.compression.type: none kafka.compression.type: none
format: json format: json
sinks:
kafka_sink:
type: kafka
properties:
topic: {{ kafka_sink_topic }}
kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.retries: 0
kafka.linger.ms: 10
kafka.request.timeout.ms: 30000
kafka.batch.size: 262144
kafka.buffer.memory: 134217728
kafka.max.request.size: 10485760
kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
format: json
json.ignore.parse.errors: false
log.failures.only: true
sinks:
clickhouse_sink: clickhouse_sink:
type: clickhouse type: clickhouse
properties: properties:
host: {{ clickhouse_sink_host }} host: "{{ clickhouse_servers }}"
table: tsg_galaxy_v3.dos_event_local table: tsg_galaxy_v3.dos_event_local
batch.size: 100000 batch.size: 100000
batch.interval: 30s batch.interval: 30s
@@ -55,10 +38,13 @@ sinks:
application: application:
env: env:
name: {{ job_name }} name: dos_event_kafka_to_clickhouse
shade.identifier: aes shade.identifier: aes
pipeline: pipeline:
object-reuse: true # [boolean] Object Reuse, default is false object-reuse: true # [boolean] Object Reuse, default is false
{{ topology }} topology:
- name: kafka_source
downstream: [clickhouse_sink]
- name: clickhouse_sink

18
tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2
View File

@@ -2,14 +2,15 @@ sources:
kafka_source: kafka_source:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_source_topic }} topic: PROXY-EVENT
kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.client.id: PROXY-EVENT
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: {{ kafka_source_group_id }} kafka.group.id: {{ kafka_source_group_id }}
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -50,6 +51,12 @@ processing_pipelines:
parameters: parameters:
value_expression: recv_time value_expression: recv_time
- function: DOMAIN
lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
output_fields: [server_domain]
parameters:
option: FIRST_SIGNIFICANT_SUBDOMAIN
- function: BASE64_DECODE_TO_STRING - function: BASE64_DECODE_TO_STRING
output_fields: [mail_subject] output_fields: [mail_subject]
parameters: parameters:
@@ -101,8 +108,9 @@ sinks:
kafka_sink: kafka_sink:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_sink_topic }} topic: PROXY-EVENT
kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.client.id: PROXY-EVENT
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -112,7 +120,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json
json.ignore.parse.errors: false json.ignore.parse.errors: false
log.failures.only: true log.failures.only: true
@@ -135,4 +143,6 @@ application:
shade.identifier: aes shade.identifier: aes
pipeline: pipeline:
object-reuse: true object-reuse: true
topology:
{{ topology }} {{ topology }}

16
tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2
View File

@@ -2,14 +2,15 @@ sources:
kafka_source: kafka_source:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_source_topic }} topic: SESSION-RECORD
kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.client.id: SESSION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: {{ kafka_source_group_id }} kafka.group.id: {{ kafka_source_group_id }}
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -50,6 +51,12 @@ processing_pipelines:
parameters: parameters:
value_expression: recv_time value_expression: recv_time
- function: DOMAIN
lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
output_fields: [server_domain]
parameters:
option: FIRST_SIGNIFICANT_SUBDOMAIN
- function: BASE64_DECODE_TO_STRING - function: BASE64_DECODE_TO_STRING
output_fields: [mail_subject] output_fields: [mail_subject]
parameters: parameters:
@@ -101,8 +108,9 @@ sinks:
kafka_sink: kafka_sink:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_sink_topic }} topic: SESSION-RECORD
kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.client.id: SESSION-RECORD
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -112,7 +120,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json
json.ignore.parse.errors: false json.ignore.parse.errors: false
log.failures.only: true log.failures.only: true

15
tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2
View File

@@ -2,14 +2,15 @@ sources:
kafka_source: kafka_source:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_source_topic }} topic: TRAFFIC-SKETCH-METRIC
kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.client.id: TRAFFIC-SKETCH-METRIC
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: etl_traffic_sketch_metric kafka.group.id: etl_traffic_sketch_metric
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
kafka.compression.type: none kafka.compression.type: none
@@ -62,12 +63,14 @@ processing_pipelines:
parameters: parameters:
data_center_id_num: 1 data_center_id_num: 1
sinks: sinks:
kafka_sink: kafka_sink:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_sink_topic }} topic: TRAFFIC-SKETCH-METRIC
kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.client.id: TRAFFIC-SKETCH-METRIC
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -77,7 +80,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json
json.ignore.parse.errors: false json.ignore.parse.errors: false
log.failures.only: true log.failures.only: true
@@ -85,7 +88,7 @@ sinks:
clickhouse_sink: clickhouse_sink:
type: clickhouse type: clickhouse
properties: properties:
host: {{ clickhouse_sink_host }} host: {{ clickhouse_servers }}
table: tsg_galaxy_v3.traffic_sketch_metric_local table: tsg_galaxy_v3.traffic_sketch_metric_local
batch.size: 100000 batch.size: 100000
batch.interval: 30s batch.interval: 30s
@@ -96,7 +99,7 @@ sinks:
application: application:
env: # [object] Environment Variables env: # [object] Environment Variables
name: {{ job_name }} name: etl_traffic_sketch_metric # [string] Job Name
shade.identifier: aes shade.identifier: aes
pipeline: pipeline:
object-reuse: true # [boolean] Object Reuse, default is false object-reuse: true # [boolean] Object Reuse, default is false

10
tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2
View File

@@ -2,14 +2,15 @@ sources:
kafka_source: kafka_source:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_source_topic }} topic: TRANSACTION-RECORD
kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.client.id: TRANSACTION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: {{ kafka_source_group_id }} kafka.group.id: {{ kafka_source_group_id }}
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -107,8 +108,9 @@ sinks:
kafka_sink: kafka_sink:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_sink_topic }} topic: TRANSACTION-RECORD
kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.client.id: TRANSACTION-RECORD
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -118,7 +120,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json
json.ignore.parse.errors: false json.ignore.parse.errors: false
log.failures.only: true log.failures.only: true

16
tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2
View File

@@ -2,14 +2,15 @@ sources:
kafka_source: kafka_source:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_source_topic }} topic: VOIP-CONVERSATION-RECORD
kafka.bootstrap.servers: {{ kafka_source_servers }} kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.client.id: VOIP-CONVERSATION-RECORD
kafka.session.timeout.ms: 60000 kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000 kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280 kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.group.id: {{ kafka_source_group_id }} kafka.group.id: {{ kafka_source_group_id }}
kafka.auto.offset.reset: latest kafka.auto.offset.reset: latest
format: json format: json
@@ -50,6 +51,12 @@ processing_pipelines:
parameters: parameters:
value_expression: recv_time value_expression: recv_time
- function: DOMAIN
lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
output_fields: [server_domain]
parameters:
option: FIRST_SIGNIFICANT_SUBDOMAIN
- function: BASE64_DECODE_TO_STRING - function: BASE64_DECODE_TO_STRING
output_fields: [mail_subject] output_fields: [mail_subject]
parameters: parameters:
@@ -101,8 +108,9 @@ sinks:
kafka_sink: kafka_sink:
type: kafka type: kafka
properties: properties:
topic: {{ kafka_sink_topic }} topic: VOIP-CONVERSATION-RECORD
kafka.bootstrap.servers: {{ kafka_sink_servers }} kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.client.id: VOIP-CONVERSATION-RECORD
kafka.retries: 0 kafka.retries: 0
kafka.linger.ms: 10 kafka.linger.ms: 10
kafka.request.timeout.ms: 30000 kafka.request.timeout.ms: 30000
@@ -112,7 +120,7 @@ sinks:
kafka.compression.type: snappy kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
format: json format: json
json.ignore.parse.errors: false json.ignore.parse.errors: false
log.failures.only: true log.failures.only: true

3
tsg_olap/upgrade/TSG-24.07/file-chunk-combiner/agg_traffic_file_chunk_combine_template
View File

@@ -5,6 +5,7 @@ flink.job.name=agg_traffic_file_chunk_combine
source.kafka.broker=192.168.44.12:9092 source.kafka.broker=192.168.44.12:9092
source.kafka.topic=TRAFFIC-FILE-STREAM-RECORD source.kafka.topic=TRAFFIC-FILE-STREAM-RECORD
source.kafka.group.id=test source.kafka.group.id=test
source.kafka.client.id=TRAFFIC-FILE-STREAM-RECORD
#earliest从头开始 latest最新 #earliest从头开始 latest最新
source.kafka.auto.offset.reset=latest source.kafka.auto.offset.reset=latest
source.kafka.session.timeout.ms=60000 source.kafka.session.timeout.ms=60000
@@ -14,7 +15,7 @@ source.kafka.max.poll.records=1000
source.kafka.max.partition.fetch.bytes=31457280 source.kafka.max.partition.fetch.bytes=31457280
source.kafka.enable.auto.commit=true source.kafka.enable.auto.commit=true
#kafka SASL验证用户名 #kafka SASL验证用户名
source.kafka.user=admin source.kafka.user=olap
#kafka SASL及SSL验证密码 #kafka SASL及SSL验证密码
source.kafka.pin=galaxy2019 source.kafka.pin=galaxy2019
#SSL需要 #SSL需要