diff --git a/tsg_olap/installation/flink/groot_stream/README.md b/tsg_olap/installation/flink/groot_stream/README.md index 619a7d6..c2ef994 100644 --- a/tsg_olap/installation/flink/groot_stream/README.md +++ b/tsg_olap/installation/flink/groot_stream/README.md @@ -1,27 +1,3 @@ -# 敏感配置信息加密 - - AES encryption support encrypt/decrypt the following parameters: - -- username -- password -- auth -- connection.user -- connection.password -- kafka.sasl.jaas.config -- kafka.ssl.keystore.password -- kafka.ssl.truststore.password -- kafka.ssl.key.password - -## Encrypt/Decrypt 举例 - -- 使用Encrypt命令对配置文件的明文敏感信息转换为密文 - - `${GROOTSTREAM_HOME}/bin/start.sh -c config/xx.yaml --encrypt ` - -- 使用Decrypt命令，将配置加密的信息转换为明文 - - `${GROOTSTREAM_HOME}/bin/start.sh -c config/xx.yaml --decrypt ` - # 配置模版举例 ## session_record.yaml.j2 （会话日志ETL场景） diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/dos_sketch_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/dos_sketch_kafka_to_ndc_kafka index 0da8393..b131c5c 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/dos_sketch_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/dos_sketch_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: DOS-SKETCH-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: DOS-SKETCH-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: dos_sketch_record_kafka_to_kafka kafka.auto.offset.reset: latest format: raw @@ -17,10 +18,11 @@ sources: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: DOS-SKETCH-RECORD kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: DOS-SKETCH-RECORD kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -30,7 +32,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_proxy_event_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_proxy_event_kafka_to_ndc_kafka index 1760cf3..512ec94 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_proxy_event_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_proxy_event_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: PROXY-EVENT kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: PROXY-EVENT kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: etl_proxy_event_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: json @@ -118,10 +119,11 @@ processing_pipelines: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: PROXY-EVENT-PROCESSED kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: PROXY-EVENT-PROCESSED kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -131,7 +133,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_session_record_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_session_record_kafka_to_ndc_kafka index 07b7e74..495b842 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_session_record_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_session_record_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: SESSION-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: SESSION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: etl_session_record_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: json @@ -117,10 +118,11 @@ processing_pipelines: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: SESSION-RECORD-PROCESSED kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: SESSION-RECORD-PROCESSED kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -130,7 +132,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_transaction_record_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_transaction_record_kafka_to_ndc_kafka index bbc0353..5f9c317 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_transaction_record_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/etl_transaction_record_kafka_to_ndc_kafka @@ -8,12 +8,13 @@ sources: properties: topic: TRANSACTION-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: TRANSACTION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: etl_transaction_record_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: json @@ -121,10 +122,11 @@ processing_pipelines: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: TRANSACTION-RECORD-PROCESSED kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: TRANSACTION-RECORD-PROCESSED kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -134,7 +136,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/network_traffic_metrics_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/network_traffic_metrics_kafka_to_ndc_kafka index 66326ad..30988dd 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/network_traffic_metrics_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/network_traffic_metrics_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: NETWORK-TRAFFIC-METRIC kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: NETWORK-TRAFFIC-METRIC kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: network_traffic_metrics_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: raw @@ -17,10 +18,11 @@ sources: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: NETWORK-TRAFFIC-METRIC kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: NETWORK-TRAFFIC-METRIC kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -30,7 +32,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/object_statistics_metric_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/object_statistics_metric_kafka_to_ndc_kafka index c6d3eb5..2a698ad 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/object_statistics_metric_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/object_statistics_metric_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: OBJECT-STATISTICS-METRIC kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: OBJECT-STATISTICS-METRIC kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: object_statistics_metric_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: raw @@ -17,9 +18,11 @@ sources: sinks: kafka_sink: - type : kafka + type: kafka properties: + topic: OBJECT-STATISTICS-METRIC kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: OBJECT-STATISTICS-METRIC kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -29,7 +32,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/policy_rule_metrics_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/policy_rule_metrics_kafka_to_ndc_kafka index 713bd65..07f2a68 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/policy_rule_metrics_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/policy_rule_metrics_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: POLICY-RULE-METRIC kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: POLICY-RULE-METRIC kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: policy_rule_metrics_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: raw @@ -17,10 +18,11 @@ sources: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: POLICY-RULE-METRIC kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: POLICY-RULE-METRIC kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -30,7 +32,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/pxy_exch_intermedia_cert_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/pxy_exch_intermedia_cert_kafka_to_ndc_kafka index fefd32d..8cca8b2 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/pxy_exch_intermedia_cert_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/pxy_exch_intermedia_cert_kafka_to_ndc_kafka @@ -4,6 +4,7 @@ sources: properties: topic: PXY-EXCH-INTERMEDIA-CERT kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: PXY-EXCH-INTERMEDIA-CERT kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 @@ -14,7 +15,7 @@ sources: kafka.ssl.truststore.password: kafka.ssl.key.password: kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.buffer.memory: kafka.group.id: pxy_exch_intermedia_cert_kafka_to_ndc_kafka kafka.auto.offset.reset: latest @@ -25,10 +26,11 @@ sources: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: PXY-EXCH-INTERMEDIA-CERT kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: PXY-EXCH-INTERMEDIA-CERT kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -38,7 +40,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/statistics_rule_metric_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/statistics_rule_metric_kafka_to_ndc_kafka index 9f247ad..ff83e39 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/statistics_rule_metric_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/statistics_rule_metric_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: STATISTICS-RULE-METRIC kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: STATISTICS-RULE-METRIC kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: statistics_rule_metric_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: raw @@ -17,10 +18,11 @@ sources: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: STATISTICS-RULE-METRIC kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: STATISTICS-RULE-METRIC kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -30,7 +32,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/troubleshooting_file_stream_record_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/troubleshooting_file_stream_record_kafka_to_ndc_kafka index 2aa7847..b880e8a 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/troubleshooting_file_stream_record_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/troubleshooting_file_stream_record_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: TROUBLESHOOTING-FILE-STREAM-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: TROUBLESHOOTING-FILE-STREAM-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: troubleshooting_file_stream_record_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: raw @@ -17,10 +18,11 @@ sources: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: TROUBLESHOOTING-FILE-STREAM-RECORD kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: TROUBLESHOOTING-FILE-STREAM-RECORD kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -30,7 +32,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/voip_record_kafka_to_ndc_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/voip_record_kafka_to_ndc_kafka index 3652211..0339db1 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/voip_record_kafka_to_ndc_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/datacenter_dt/voip_record_kafka_to_ndc_kafka @@ -4,12 +4,13 @@ sources: properties: topic: VOIP-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: VOIP-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: statistics_rule_metric_kafka_to_ndc_kafka kafka.auto.offset.reset: latest format: json @@ -55,10 +56,11 @@ processing_pipelines: sinks: kafka_sink: - type : kafka + type: kafka properties: topic: VOIP-RECORD kafka.bootstrap.servers: "{{ kafka_sink_servers }}" + kafka.client.id: VOIP-RECORD kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -68,7 +70,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/dos_event_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/dos_event_kafka_to_clickhouse index 195b8ec..2800bf2 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/dos_event_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/dos_event_kafka_to_clickhouse @@ -4,12 +4,13 @@ sources: properties: topic: DOS-EVENT kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: DOS-EVENT kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: dos_event_kafka_to_clickhouse kafka.auto.offset.reset: latest format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_session_record_processed_kafka_to_cn_kafka b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_session_record_processed_kafka_to_cn_kafka index 82a30a6..be5490c 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_session_record_processed_kafka_to_cn_kafka +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_session_record_processed_kafka_to_cn_kafka @@ -4,12 +4,13 @@ sources: properties: # [object] Source Properties topic: SESSION-RECORD-PROCESSED kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: SESSION-RECORD-PROCESSED kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: etl_processed_session_record_kafka_to_cn_kafka kafka.auto.offset.reset: latest format: json @@ -362,6 +363,7 @@ sinks: properties: topic: SESSION-RECORD-CN kafka.bootstrap.servers: {{ national_center_cn_kafka_servers }} + kafka.client.id: SESSION-RECORD-CN kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -371,7 +373,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_voip_record_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_voip_record_kafka_to_clickhouse index da47689..7b46dc8 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_voip_record_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/etl_voip_record_kafka_to_clickhouse @@ -4,12 +4,13 @@ sources: properties: topic: VOIP-CONVERSATION-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: VOIP-CONVERSATION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: etl_voip_record_kafka_to_clickhouse kafka.auto.offset.reset: latest format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/proxy_event_processed_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/proxy_event_processed_kafka_to_clickhouse index fc48f64..8cdfae5 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/proxy_event_processed_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/proxy_event_processed_kafka_to_clickhouse @@ -4,12 +4,13 @@ sources: properties: topic: PROXY-EVENT-PROCESSED kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: PROXY-EVENT-PROCESSED kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: proxy_event_processed_kafka_to_clickhouse kafka.auto.offset.reset: latest format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/session_record_processed_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/session_record_processed_kafka_to_clickhouse index 2798840..7d0f68b 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/session_record_processed_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/session_record_processed_kafka_to_clickhouse @@ -4,12 +4,13 @@ sources: properties: topic: SESSION-RECORD-PROCESSED kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: SESSION-RECORD-PROCESSED kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: session_record_processed_kafka_to_clickhouse kafka.auto.offset.reset: latest format: json diff --git a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/transaction_record_processed_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/transaction_record_processed_kafka_to_clickhouse index 9aeb79d..83ce33d 100644 --- a/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/transaction_record_processed_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/multi-datacenter-examples/national_datacenter/transaction_record_processed_kafka_to_clickhouse @@ -4,12 +4,13 @@ sources: properties: topic: TRANSACTION-RECORD-PROCESSED kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: TRANSACTION-RECORD-PROCESSED kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: transaction_record_processed_kafka_to_clickhouse kafka.auto.offset.reset: latest format: json diff --git a/tsg_olap/installation/flink/groot_stream/realtime_log_streaming_cn_session_record.yaml.template b/tsg_olap/installation/flink/groot_stream/realtime_log_streaming_cn_session_record.yaml.template index 2a011a2..3e9db4e 100644 --- a/tsg_olap/installation/flink/groot_stream/realtime_log_streaming_cn_session_record.yaml.template +++ b/tsg_olap/installation/flink/groot_stream/realtime_log_streaming_cn_session_record.yaml.template @@ -4,12 +4,13 @@ sources: properties: topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_bootstrap_servers }} + kafka.client.id: {{ kafka_source_topic }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: {{ kafka_source_group_id }} kafka.auto.offset.reset: latest format: json @@ -354,6 +355,7 @@ sinks: properties: topic: {{ kafka_sink_topic }} kafka.bootstrap.servers: {{ kafka_sink_bootstrap_servers }} + kafka.client.id: {{ kafka_sink_topic }} kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -363,7 +365,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json json.ignore.parse.errors: false log.failures.only: true diff --git a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/dos_event_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/dos_event_kafka_to_clickhouse index 49a3d1a..5152734 100644 --- a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/dos_event_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/dos_event_kafka_to_clickhouse @@ -4,6 +4,7 @@ sources: properties: topic: DOS-EVENT kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: DOS-EVENT kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 @@ -14,7 +15,7 @@ sources: kafka.ssl.truststore.password: kafka.ssl.key.password: kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.buffer.memory: kafka.group.id: dos_event_kafka_to_clickhouse-20231221 kafka.auto.offset.reset: latest diff --git a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_datapath_telemetry_record_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_datapath_telemetry_record_kafka_to_clickhouse index 8ffbda3..9ae5b54 100644 --- a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_datapath_telemetry_record_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_datapath_telemetry_record_kafka_to_clickhouse @@ -4,6 +4,7 @@ sources: properties: topic: DATAPATH-TELEMETRY-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: DATAPATH-TELEMETRY-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 @@ -16,7 +17,7 @@ sources: #kafka.ssl.key.password: 86cf0e2ffba3f541a6c6761313e5cc7e kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: etl_datapath_telemetry_record_kafka_to_clickhouse-20230125 kafka.auto.offset.reset: latest format: msgpack diff --git a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_proxy_event_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_proxy_event_kafka_to_clickhouse index 19e4560..011eabb 100644 --- a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_proxy_event_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_proxy_event_kafka_to_clickhouse @@ -8,6 +8,7 @@ sources: properties: topic: PROXY-EVENT kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: PROXY-EVENT kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 @@ -18,7 +19,7 @@ sources: kafka.ssl.truststore.password: kafka.ssl.key.password: kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.buffer.memory: kafka.group.id: etl_proxy_event_kafka_to_clickhouse-20231221 kafka.auto.offset.reset: latest diff --git a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_session_record_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_session_record_kafka_to_clickhouse index ada2632..94186d9 100644 --- a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_session_record_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_session_record_kafka_to_clickhouse @@ -8,6 +8,7 @@ sources: properties: topic: SESSION-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: SESSION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 @@ -18,7 +19,7 @@ sources: kafka.ssl.truststore.password: kafka.ssl.key.password: kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.buffer.memory: kafka.group.id: etl_session_record_kafka_to_clickhouse-20230125 kafka.auto.offset.reset: latest diff --git a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_traffic_sketch_metric_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_traffic_sketch_metric_kafka_to_clickhouse index 016650c..9eee8c4 100644 --- a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_traffic_sketch_metric_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_traffic_sketch_metric_kafka_to_clickhouse @@ -4,12 +4,13 @@ sources: properties: topic: TRAFFIC-SKETCH-METRIC kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: TRAFFIC-SKETCH-METRIC kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: etl_traffic_sketch_metric kafka.auto.offset.reset: latest kafka.compression.type: none diff --git a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_transaction_record_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_transaction_record_kafka_to_clickhouse index d4dbd87..03bd6d7 100644 --- a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_transaction_record_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_transaction_record_kafka_to_clickhouse @@ -8,6 +8,7 @@ sources: properties: topic: TRANSACTION-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: TRANSACTION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 @@ -18,7 +19,7 @@ sources: kafka.ssl.truststore.password: kafka.ssl.key.password: kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.buffer.memory: kafka.group.id: etl_transaction_record_kafka_to_clickhouse-20240308 kafka.auto.offset.reset: latest diff --git a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_voip_record_kafka_to_clickhouse b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_voip_record_kafka_to_clickhouse index d798e94..6fde822 100644 --- a/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_voip_record_kafka_to_clickhouse +++ b/tsg_olap/installation/flink/groot_stream/single-cluster-examples/etl_voip_record_kafka_to_clickhouse @@ -8,6 +8,7 @@ sources: properties: topic: VOIP-CONVERSATION-RECORD kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: VOIP-CONVERSATION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 @@ -18,7 +19,7 @@ sources: kafka.ssl.truststore.password: kafka.ssl.key.password: kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.buffer.memory: kafka.group.id: etl_voip_record_kafka_to_clickhouse-20231221 kafka.auto.offset.reset: latest diff --git a/tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2 index 5ad1d25..5bb3aa3 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2 @@ -4,12 +4,13 @@ sources: properties: topic: {{ kafka_source_topic }} kafka.bootstrap.servers: {{ kafka_source_servers }} + kafka.client.id: {{ kafka_source_topic }} kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: {{ kafka_source_group_id }} kafka.auto.offset.reset: latest format: raw @@ -19,7 +20,8 @@ sinks: type: kafka properties: topic: {{ kafka_sink_topic }} - kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.bootstrap.servers: { { kafka_sink_servers } } + kafka.client.id: {{ kafka_sink_topic }} kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -29,13 +31,13 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw clickhouse_sink: type: clickhouse properties: - host: {{ clickhouse_sink_host }} + host: {{ clickhouse_servers }} table: tsg_galaxy_v3.traffic_sketch_metric_local batch.size: 100000 batch.interval: 30s @@ -48,6 +50,7 @@ application: shade.identifier: aes pipeline: object-reuse: true + topology: {{ topology }} diff --git a/tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2 index 06058b8..2d64c1f 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2 @@ -2,14 +2,15 @@ sources: kafka_source: type: kafka properties: - topic: {{ kafka_source_topic }} + topic: DATAPATH-TELEMETRY-RECORD kafka.bootstrap.servers: {{ kafka_source_servers }} + kafka.client.id: DATAPATH-TELEMETRY-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: {{ kafka_source_group_id }} kafka.auto.offset.reset: latest format: msgpack @@ -37,8 +38,9 @@ sinks: kafka_sink: type: kafka properties: - topic: {{ kafka_sink_topic }} + topic: DATAPATH-TELEMETRY-RECORD kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.client.id: DATAPATH-TELEMETRY-RECORD kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -48,7 +50,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: raw json.ignore.parse.errors: false log.failures.only: true @@ -71,6 +73,7 @@ application: shade.identifier: aes pipeline: object-reuse: true + topology: {{ topology }} diff --git a/tsg_olap/installation/flink/groot_stream/templates/dos_event.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/dos_event_kafka_to_clickhouse similarity index 54% rename from tsg_olap/installation/flink/groot_stream/templates/dos_event.yaml.j2 rename to tsg_olap/installation/flink/groot_stream/templates/dos_event_kafka_to_clickhouse index e44b182..5152734 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/dos_event.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/dos_event_kafka_to_clickhouse @@ -2,8 +2,9 @@ sources: kafka_source: type: kafka properties: - topic: {{ kafka_source_topic }} + topic: DOS-EVENT kafka.bootstrap.servers: "{{ kafka_source_servers }}" + kafka.client.id: DOS-EVENT kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 @@ -14,7 +15,7 @@ sources: kafka.ssl.truststore.password: kafka.ssl.key.password: kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.buffer.memory: kafka.group.id: dos_event_kafka_to_clickhouse-20231221 kafka.auto.offset.reset: latest @@ -22,30 +23,12 @@ sources: kafka.compression.type: none format: json -sinks: - kafka_sink: - type: kafka - properties: - topic: {{ kafka_sink_topic }} - kafka.bootstrap.servers: {{ kafka_sink_servers }} - kafka.retries: 0 - kafka.linger.ms: 10 - kafka.request.timeout.ms: 30000 - kafka.batch.size: 262144 - kafka.buffer.memory: 134217728 - kafka.max.request.size: 10485760 - kafka.compression.type: snappy - kafka.security.protocol: SASL_PLAINTEXT - kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 - format: json - json.ignore.parse.errors: false - log.failures.only: true +sinks: clickhouse_sink: type: clickhouse properties: - host: {{ clickhouse_sink_host }} + host: "{{ clickhouse_servers }}" table: tsg_galaxy_v3.dos_event_local batch.size: 100000 batch.interval: 30s @@ -55,10 +38,13 @@ sinks: application: env: - name: {{ job_name }} + name: dos_event_kafka_to_clickhouse shade.identifier: aes pipeline: object-reuse: true # [boolean] Object Reuse, default is false - {{ topology }} + topology: + - name: kafka_source + downstream: [clickhouse_sink] + - name: clickhouse_sink diff --git a/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 index 6c750e2..716667d 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2 @@ -2,14 +2,15 @@ sources: kafka_source: type: kafka properties: - topic: {{ kafka_source_topic }} + topic: PROXY-EVENT kafka.bootstrap.servers: {{ kafka_source_servers }} + kafka.client.id: PROXY-EVENT kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: {{ kafka_source_group_id }} kafka.auto.offset.reset: latest format: json @@ -50,6 +51,12 @@ processing_pipelines: parameters: value_expression: recv_time + - function: DOMAIN + lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] + output_fields: [server_domain] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -101,8 +108,9 @@ sinks: kafka_sink: type: kafka properties: - topic: {{ kafka_sink_topic }} + topic: PROXY-EVENT kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.client.id: PROXY-EVENT kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -112,7 +120,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json json.ignore.parse.errors: false log.failures.only: true @@ -135,4 +143,6 @@ application: shade.identifier: aes pipeline: object-reuse: true + topology: {{ topology }} + diff --git a/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 index 7970a95..6cbbc40 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2 @@ -2,14 +2,15 @@ sources: kafka_source: type: kafka properties: - topic: {{ kafka_source_topic }} + topic: SESSION-RECORD kafka.bootstrap.servers: {{ kafka_source_servers }} + kafka.client.id: SESSION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: {{ kafka_source_group_id }} kafka.auto.offset.reset: latest format: json @@ -50,6 +51,12 @@ processing_pipelines: parameters: value_expression: recv_time + - function: DOMAIN + lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] + output_fields: [server_domain] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -101,8 +108,9 @@ sinks: kafka_sink: type: kafka properties: - topic: {{ kafka_sink_topic }} + topic: SESSION-RECORD kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.client.id: SESSION-RECORD kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -112,7 +120,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json json.ignore.parse.errors: false log.failures.only: true diff --git a/tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2 index 571c1af..5e49ac6 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2 @@ -2,14 +2,15 @@ sources: kafka_source: type: kafka properties: - topic: {{ kafka_source_topic }} + topic: TRAFFIC-SKETCH-METRIC kafka.bootstrap.servers: {{ kafka_source_servers }} + kafka.client.id: TRAFFIC-SKETCH-METRIC kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: etl_traffic_sketch_metric kafka.auto.offset.reset: latest kafka.compression.type: none @@ -62,12 +63,14 @@ processing_pipelines: parameters: data_center_id_num: 1 + sinks: kafka_sink: type: kafka properties: - topic: {{ kafka_sink_topic }} + topic: TRAFFIC-SKETCH-METRIC kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.client.id: TRAFFIC-SKETCH-METRIC kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -77,7 +80,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json json.ignore.parse.errors: false log.failures.only: true @@ -85,7 +88,7 @@ sinks: clickhouse_sink: type: clickhouse properties: - host: {{ clickhouse_sink_host }} + host: {{ clickhouse_servers }} table: tsg_galaxy_v3.traffic_sketch_metric_local batch.size: 100000 batch.interval: 30s @@ -96,7 +99,7 @@ sinks: application: env: # [object] Environment Variables - name: {{ job_name }} + name: etl_traffic_sketch_metric # [string] Job Name shade.identifier: aes pipeline: object-reuse: true # [boolean] Object Reuse, default is false diff --git a/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 index 49d1782..4886509 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/transaction_record.yaml.j2 @@ -2,14 +2,15 @@ sources: kafka_source: type: kafka properties: - topic: {{ kafka_source_topic }} + topic: TRANSACTION-RECORD kafka.bootstrap.servers: {{ kafka_source_servers }} + kafka.client.id: TRANSACTION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: {{ kafka_source_group_id }} kafka.auto.offset.reset: latest format: json @@ -107,8 +108,9 @@ sinks: kafka_sink: type: kafka properties: - topic: {{ kafka_sink_topic }} + topic: TRANSACTION-RECORD kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.client.id: TRANSACTION-RECORD kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -118,7 +120,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json json.ignore.parse.errors: false log.failures.only: true diff --git a/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 b/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 index a37d469..64627a5 100644 --- a/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 +++ b/tsg_olap/installation/flink/groot_stream/templates/voip_record.yaml.j2 @@ -2,14 +2,15 @@ sources: kafka_source: type: kafka properties: - topic: {{ kafka_source_topic }} + topic: VOIP-CONVERSATION-RECORD kafka.bootstrap.servers: {{ kafka_source_servers }} + kafka.client.id: VOIP-CONVERSATION-RECORD kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 kafka.group.id: {{ kafka_source_group_id }} kafka.auto.offset.reset: latest format: json @@ -50,6 +51,12 @@ processing_pipelines: parameters: value_expression: recv_time + - function: DOMAIN + lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni] + output_fields: [server_domain] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + - function: BASE64_DECODE_TO_STRING output_fields: [mail_subject] parameters: @@ -101,8 +108,9 @@ sinks: kafka_sink: type: kafka properties: - topic: {{ kafka_sink_topic }} + topic: VOIP-CONVERSATION-RECORD kafka.bootstrap.servers: {{ kafka_sink_servers }} + kafka.client.id: VOIP-CONVERSATION-RECORD kafka.retries: 0 kafka.linger.ms: 10 kafka.request.timeout.ms: 30000 @@ -112,7 +120,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json json.ignore.parse.errors: false log.failures.only: true diff --git a/tsg_olap/upgrade/TSG-24.07/file-chunk-combiner/agg_traffic_file_chunk_combine_template b/tsg_olap/upgrade/TSG-24.07/file-chunk-combiner/agg_traffic_file_chunk_combine_template index f0f9704..a5d077b 100644 --- a/tsg_olap/upgrade/TSG-24.07/file-chunk-combiner/agg_traffic_file_chunk_combine_template +++ b/tsg_olap/upgrade/TSG-24.07/file-chunk-combiner/agg_traffic_file_chunk_combine_template @@ -5,6 +5,7 @@ flink.job.name=agg_traffic_file_chunk_combine source.kafka.broker=192.168.44.12:9092 source.kafka.topic=TRAFFIC-FILE-STREAM-RECORD source.kafka.group.id=test +source.kafka.client.id=TRAFFIC-FILE-STREAM-RECORD #earliest从头开始 latest最新 source.kafka.auto.offset.reset=latest source.kafka.session.timeout.ms=60000 @@ -14,7 +15,7 @@ source.kafka.max.poll.records=1000 source.kafka.max.partition.fetch.bytes=31457280 source.kafka.enable.auto.commit=true #kafka SASL验证用户名 -source.kafka.user=admin +source.kafka.user=olap #kafka SASL及SSL验证密码 source.kafka.pin=galaxy2019 #SSL需要