117 lines
6.0 KiB
JSON
117 lines
6.0 KiB
JSON
{
|
|
"type": "kafka",
|
|
"dataSchema": {
|
|
"dataSource": "statistics_rule",
|
|
"parser": {
|
|
"type": "string",
|
|
"parseSpec": {
|
|
"format": "json",
|
|
"timestampSpec": {
|
|
"column": "timestamp_ms",
|
|
"format": "millis"
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [
|
|
"device_id",
|
|
"device_group",
|
|
"data_center",
|
|
{"name": "vsys_id", "type": "long"},
|
|
{"name": "rule_id", "type": "long"},
|
|
{"name": "template_id", "type": "long"},
|
|
{"name": "chart_id", "type": "long"},
|
|
{"name": "version", "type": "long"},
|
|
{"name": "client_ip_object_list", "type": "string", "multiValueHandling": "SORTED_SET"},
|
|
{"name": "server_ip_object_list", "type": "string", "multiValueHandling": "SORTED_SET"},
|
|
{"name": "fqdn_category_list", "type": "string", "multiValueHandling": "SORTED_SET"},
|
|
"client_ip",
|
|
"server_ip",
|
|
"client_country",
|
|
"server_country",
|
|
{"name": "client_asn", "type": "long"},
|
|
{"name": "server_asn", "type": "long"},
|
|
"server_fqdn",
|
|
"server_domain",
|
|
"app",
|
|
"app_category",
|
|
{"name": "c2s_ttl", "type": "long"},
|
|
{"name": "s2c_ttl", "type": "long"},
|
|
{"name": "c2s_link_id", "type": "long"},
|
|
{"name": "s2c_link_id", "type": "long"},
|
|
{"name": "client_port", "type": "long"},
|
|
{"name": "server_port", "type": "long"},
|
|
"ssl_ja3_hash",
|
|
"http_host",
|
|
"ssl_sni",
|
|
"subscriber_id",
|
|
"decoded_as",
|
|
"sled_ip",
|
|
{"name": "flags", "type": "long"},
|
|
"direction",
|
|
"bucket_id"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"metricsSpec": [
|
|
{"type": "longSum", "name": "count_1", "fieldName": "count_1"},
|
|
{"type": "longSum", "name": "count_2", "fieldName": "count_2"},
|
|
{"type": "longSum", "name": "count_3", "fieldName": "count_3"},
|
|
{"type": "longSum", "name": "count_4", "fieldName": "count_4"},
|
|
{"type": "HLLDSketchMerge", "name": "unique_count_1", "fieldName": "unique_count_1"},
|
|
{"type": "HLLDSketchMerge", "name": "unique_count_2", "fieldName": "unique_count_2"},
|
|
{"type": "HLLDSketchMerge", "name": "unique_count_3", "fieldName": "unique_count_3"},
|
|
{"type": "HLLDSketchMerge", "name": "unique_count_4", "fieldName": "unique_count_4"},
|
|
{"type": "HdrHistogramSketchMerge", "name": "distribution_1", "fieldName": "distribution_1"},
|
|
{"type": "HdrHistogramSketchMerge", "name": "distribution_2", "fieldName": "distribution_2"},
|
|
{"type": "HdrHistogramSketchMerge", "name": "distribution_3", "fieldName": "distribution_3"},
|
|
{"type": "HdrHistogramSketchMerge", "name": "distribution_4", "fieldName": "distribution_4"}
|
|
],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {"type": "period", "period": "PT1S"},
|
|
"rollup": true
|
|
},
|
|
"transformSpec": {
|
|
"transforms": [
|
|
{"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
|
|
{"type": "expression", "name": "client_ip_object_list", "expression": "STRING_TO_ARRAY(client_ip_object_list,',')"},
|
|
{"type": "expression", "name": "server_ip_object_list", "expression": "STRING_TO_ARRAY(server_ip_object_list,',')"},
|
|
{"type": "expression", "name": "fqdn_category_list", "expression": "STRING_TO_ARRAY(fqdn_category_list,',')"},
|
|
{"type": "expression", "name": "count_1", "expression": "nvl(count_1, 0)"},
|
|
{"type": "expression", "name": "count_2", "expression": "nvl(count_2, 0)"},
|
|
{"type": "expression", "name": "count_3", "expression": "nvl(count_3, 0)"},
|
|
{"type": "expression", "name": "count_4", "expression": "nvl(count_4, 0)"},
|
|
{"type": "expression", "name": "bucket_id", "expression": "dimension_bucket(1024,device_id,rule_id,template_id,chart_id,version,client_ip_object_list,server_ip_object_list,fqdn_category_list,client_ip,server_ip,client_country,server_country,client_asn,server_asn,server_fqdn,server_domain,app,app_category,c2s_ttl,s2c_ttl,c2s_link_id,s2c_link_id,client_port,server_port,ssl_ja3_hash,http_host,ssl_sni,subscriber_id,decoded_as,sled_ip,flags,direction)"}
|
|
],
|
|
"filter": {"type": "selector", "dimension": "name", "value": "statistics_rule"}
|
|
}
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxRowsInMemory": 250000,
|
|
"maxBytesInMemory": -1,
|
|
"resetOffsetAutomatically": true,
|
|
"reportParseExceptions": false
|
|
},
|
|
"ioConfig": {
|
|
"topic": "STATISTICS-RULE-METRIC",
|
|
"taskCount": 2,
|
|
"replicas": 1,
|
|
"taskDuration": "PT1H",
|
|
"completionTimeout": "PT30M",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafkabootstrap",
|
|
"sasl.mechanism": "PLAIN",
|
|
"security.protocol": "SASL_PLAINTEXT",
|
|
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
|
|
}
|
|
},
|
|
"context":{
|
|
"druid.indexer.runner.javaOpts": "-server -Xms4g -Xmx4g -XX:MaxDirectMemorySize=2g"
|
|
}
|
|
}
|
|
|
|
|