去除5分钟聚合逻辑,只取消嵌套格式

This commit is contained in:
wangkuan
2024-08-30 11:43:18 +08:00
parent 94a2aaaed7
commit 3df8fa0176
2 changed files with 12 additions and 104 deletions

View File

@@ -4,13 +4,12 @@ sources:
properties:
topic: TRAFFIC-SKETCH-METRIC
kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.client.id: TRAFFIC-SKETCH-METRIC
kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
kafka.group.id: etl_traffic_sketch_metric
kafka.auto.offset.reset: latest
kafka.compression.type: none
@@ -18,29 +17,16 @@ sources:
processing_pipelines:
etl_processor: # [object] Processing Pipeline
type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
type: projection
remove_fields:
output_fields:
functions: # [array of object] Function List
- function: FLATTEN
lookup_fields: [ fields,tags ]
output_fields: [ ]
- function: UNIX_TIMESTAMP_CONVERTER
lookup_fields: [ timestamp_ms ]
output_fields: [ recv_time ]
parameters:
#prefix: ""
depth: 3
# delimiter: "."
- function: RENAME
lookup_fields: [ '' ]
output_fields: [ '' ]
filter:
parameters:
# parent_fields: [tags]
#rename_fields:
# tags: tags
rename_expression: key =string.replace_all(key,'tags.','');key =string.replace_all(key,'fields.','');return key;
precision: seconds
interval: 60
- function: EVAL
output_fields: [ internal_ip ]
parameters:
@@ -49,13 +35,6 @@ processing_pipelines:
output_fields: [ external_ip ]
parameters:
value_expression: 'direction=Outbound? server_ip : client_ip'
- function: UNIX_TIMESTAMP_CONVERTER
lookup_fields: [ timestamp_ms ]
output_fields: [ recv_time ]
parameters:
precision: seconds
- function: SNOWFLAKE_ID
lookup_fields: [ '' ]
output_fields: [ log_id ]
@@ -70,7 +49,6 @@ sinks:
properties:
topic: TRAFFIC-SKETCH-METRIC
kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.client.id: TRAFFIC-SKETCH-METRIC
kafka.retries: 0
kafka.linger.ms: 10
kafka.request.timeout.ms: 30000
@@ -105,3 +83,4 @@ application:
object-reuse: true # [boolean] Object Reuse, default is false
{{ topology }}