gfwleak/galaxy-deployment-tsg-olap-data-initialization
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加24.09Druid任务和安装包全局配置

Browse Source
This commit is contained in:
zhanghongqing
2024-11-12 18:17:04 +08:00
parent aca3f7f46b
commit 1346da854c
26 changed files with 2169 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
druid/templates/tasks/index_kafka_application_protocol_stat.json Normal file
View File

@@ -0,0 +1,102 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "application_protocol_stat",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"device_group",
"data_center",
"protocol_stack_id",
"app_name",
{"name": "vsys_id", "type": "long"}
]
}
}
},
"metricsSpec": [
{"type": "longSum", "name": "sessions", "fieldName": "sessions"},
{"type": "longSum", "name": "c2s_pkts", "fieldName": "c2s_pkts"},
{"type": "longSum", "name": "s2c_pkts", "fieldName": "s2c_pkts"},
{"type": "longSum", "name": "c2s_bytes", "fieldName": "c2s_bytes"},
{"type": "longSum", "name": "s2c_bytes", "fieldName": "s2c_bytes"},
{"type": "longSum", "name": "c2s_fragments", "fieldName": "c2s_fragments"},
{"type": "longSum", "name": "s2c_fragments", "fieldName": "s2c_fragments"},
{"type": "longSum", "name": "c2s_tcp_lost_bytes", "fieldName": "c2s_tcp_lost_bytes"},
{"type": "longSum", "name": "s2c_tcp_lost_bytes", "fieldName": "s2c_tcp_lost_bytes"},
{"type": "longSum", "name": "c2s_tcp_ooorder_pkts", "fieldName": "c2s_tcp_ooorder_pkts"},
{"type": "longSum", "name": "s2c_tcp_ooorder_pkts", "fieldName": "s2c_tcp_ooorder_pkts"},
{"type": "longSum", "name": "c2s_tcp_retransmitted_pkts", "fieldName": "c2s_tcp_retransmitted_pkts"},
{"type": "longSum", "name": "s2c_tcp_retransmitted_pkts", "fieldName": "s2c_tcp_retransmitted_pkts"},
{"type": "longSum", "name": "c2s_tcp_retransmitted_bytes", "fieldName": "c2s_tcp_retransmitted_bytes"},
{"type": "longSum", "name": "s2c_tcp_retransmitted_bytes", "fieldName": "s2c_tcp_retransmitted_bytes"},
{"type": "longSum", "name": "in_bytes", "fieldName": "in_bytes"},
{"type": "longSum", "name": "out_bytes", "fieldName": "out_bytes"},
{"type": "longSum", "name": "in_pkts", "fieldName": "in_pkts"},
{"type": "longSum", "name": "out_pkts", "fieldName": "out_pkts"}
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec": {
"transforms": [
{"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "c2s_pkts", "expression": "nvl(c2s_pkts, 0)"},
{"type": "expression", "name": "s2c_pkts", "expression": "nvl(s2c_pkts, 0)"},
{"type": "expression", "name": "c2s_bytes", "expression": "nvl(c2s_bytes, 0)"},
{"type": "expression", "name": "s2c_bytes", "expression": "nvl(s2c_bytes, 0)"},
{"type": "expression", "name": "c2s_fragments", "expression": "nvl(c2s_fragments, 0)"},
{"type": "expression", "name": "s2c_fragments", "expression": "nvl(s2c_fragments, 0)"},
{"type": "expression", "name": "c2s_tcp_lost_bytes", "expression": "nvl(c2s_tcp_lost_bytes, 0)"},
{"type": "expression", "name": "s2c_tcp_lost_bytes", "expression": "nvl(s2c_tcp_lost_bytes, 0)"},
{"type": "expression", "name": "c2s_tcp_ooorder_pkts", "expression": "nvl(c2s_tcp_ooorder_pkts, 0)"},
{"type": "expression", "name": "s2c_tcp_ooorder_pkts", "expression": "nvl(s2c_tcp_ooorder_pkts, 0)"},
{"type": "expression", "name": "c2s_tcp_retransmitted_pkts", "expression": "nvl(c2s_tcp_retransmitted_pkts, 0)"},
{"type": "expression", "name": "s2c_tcp_retransmitted_pkts", "expression": "nvl(s2c_tcp_retransmitted_pkts, 0)"},
{"type": "expression", "name": "c2s_tcp_retransmitted_bytes", "expression": "nvl(c2s_tcp_retransmitted_bytes, 0)"},
{"type": "expression", "name": "s2c_tcp_retransmitted_bytes", "expression": "nvl(s2c_tcp_retransmitted_bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {
"type": "selector",
"dimension": "name",
"value": "application_protocol_stat"
}
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsInMemory": 200000,
"maxRowsPerSegment": 5000000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "NETWORK-TRAFFIC-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

68
druid/templates/tasks/index_kafka_dos_protection_rule_hits.json Normal file
View File

@@ -0,0 +1,68 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "dos_protection_rule_hits",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"device_group",
"data_center",
{"name": "vsys_id","type": "long"},
{"name": "rule_id","type": "long"}
]
}
}
},
"metricsSpec": [
{"type": "longSum", "name": "in_bytes", "fieldName": "in_bytes"},
{"type": "longSum", "name": "out_bytes", "fieldName": "out_bytes"},
{"type": "longSum", "name": "hit_count", "fieldName": "hit_count"},
{"type": "longSum", "name": "in_pkts", "fieldName": "in_pkts"},
{"type": "longSum", "name": "out_pkts", "fieldName": "out_pkts"}
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec": {
"transforms": [
{"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector","dimension": "name", "value": "dos_protection_rule_hits"}
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "POLICY-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

68
druid/templates/tasks/index_kafka_dos_protection_rule_metric.json Normal file
View File

@@ -0,0 +1,68 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "dos_protection_rule_metric",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
{"name": "rule_id","type": "long"},
"name",
"client_ip",
"client_country",
"server_ip",
"server_country"
]
}
}
},
"metricsSpec": [
{"type": "longSum", "name": "sessions", "fieldName": "sessions"},
{"type": "longSum", "name": "pkts", "fieldName": "pkts"},
{"type": "longSum", "name": "bytes", "fieldName": "bytes"}
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec": {
"transforms": [
{"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"}
]
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "DOS-PROTECTION-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

61
druid/templates/tasks/index_kafka_dos_sketch_top_server_ip.json Normal file
View File

@@ -0,0 +1,61 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "dos_sketch_top_server_ip",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "sketch_start_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id", "type": "long"},
"destination_ip",
"attack_type",
{"type": "long", "name": "partition_num"}
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"},
{ "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" },
{ "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT10M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression","name": "vsys_id","expression": "nvl(vsys_id,1)"},
{"type": "expression", "name": "session_rate", "expression": "nvl(session_rate, 0)"},
{"type": "expression", "name": "packet_rate", "expression": "nvl(packet_rate, 0)"},
{"type": "expression", "name": "bit_rate", "expression": "nvl(bit_rate, 0)"}
]
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "DOS-SKETCH-TOP-SERVER-IP",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

67
druid/templates/tasks/index_kafka_monitor_rule_hits.json Normal file
View File

@@ -0,0 +1,67 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "monitor_rule_hits",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
{"name": "rule_id","type": "long"},
{"name": "action","type": "long"}
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": { "type": "selector", "dimension": "name", "value": "monitor_rule_hits" }
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "POLICY-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

75
druid/templates/tasks/index_kafka_object_statistics.json Normal file
View File

@@ -0,0 +1,75 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "object_statistics",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
{"name": "template_id","type": "long"},
{"name": "chart_id","type": "long"},
{"name": "version","type": "long"},
"object_type",
{"name": "object_id","type": "long"},
{"name": "item_id","type": "long"}
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "new_in_sessions", "fieldName" : "new_in_sessions" },
{ "type" : "longSum", "name" : "new_out_sessions", "fieldName" : "new_out_sessions" },
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"},
{"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"},
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}
],
"filter": { "type": "selector", "dimension": "name", "value": "object_statistics" }
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsInMemory": 250000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "OBJECT-STATISTICS-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

72
druid/templates/tasks/index_kafka_proxy_rule_hits.json Normal file
View File

@@ -0,0 +1,72 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "proxy_rule_hits",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"device_group",
"data_center",
{ "name": "vsys_id", "type": "long" },
{ "name": "rule_id", "type": "long" },
{ "name": "pinning_status", "type": "string" },
{ "name": "action", "type": "long" },
{ "name": "sub_action", "type": "string" }
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": { "type": "selector", "dimension": "name", "value": "proxy_rule_hits" }
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "POLICY-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

67
druid/templates/tasks/index_kafka_security_rule_hits.json Normal file
View File

@@ -0,0 +1,67 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "security_rule_hits",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
{"name": "rule_id","type": "long"},
{"name": "action","type": "long"}
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": { "type": "selector", "dimension": "name", "value": "security_rule_hits" }
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "POLICY-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

68
druid/templates/tasks/index_kafka_service_chaining_rule_hits.json Normal file
View File

@@ -0,0 +1,68 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "service_chaining_rule_hits",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"device_group",
"data_center",
{ "name": "vsys_id", "type": "long" },
{ "name": "rule_id", "type": "long" },
{ "name": "sff_profile_id", "type": "long" },
{ "name": "sf_profile_id", "type": "long" }
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sent_pkts", "fieldName" : "sent_pkts" },
{ "type" : "longSum", "name" : "sent_bytes", "fieldName" : "sent_bytes" },
{ "type" : "longSum", "name" : "recv_pkts", "fieldName" : "recv_pkts" },
{ "type" : "longSum", "name" : "recv_bytes", "fieldName" : "recv_bytes" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sent_pkts", "expression": "nvl(sent_pkts, 0)"},
{"type": "expression", "name": "sent_bytes", "expression": "nvl(sent_bytes, 0)"},
{"type": "expression", "name": "recv_pkts", "expression": "nvl(recv_pkts, 0)"},
{"type": "expression", "name": "recv_bytes", "expression": "nvl(recv_bytes, 0)"}
],
"filter": { "type": "selector", "dimension": "name", "value": "service_chaining_rule_hits" }
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "POLICY-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

62
druid/templates/tasks/index_kafka_service_function_status.json Normal file
View File

@@ -0,0 +1,62 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "service_function_status",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"device_group",
"data_center",
{ "name": "vsys_id", "type": "long" },
{ "name": "sf_profile_id", "type": "long" }
]
}
}
},
"metricsSpec" : [
{ "type" : "longMax", "name" : "sf_latency_us", "fieldName" : "sf_latency_us" },
{ "type" : "longMax", "name" : "sf_status", "fieldName" : "sf_status" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sf_latency_us", "expression": "nvl(sf_latency_us, 0)"},
{"type": "expression", "name": "sf_status", "expression": "nvl(sf_status, 0)"}
],
"filter": { "type": "selector", "dimension": "name", "value": "service_function_status" }
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "POLICY-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

116
druid/templates/tasks/index_kafka_statistics_rule.json Normal file
View File

@@ -0,0 +1,116 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "statistics_rule",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"device_group",
"data_center",
{"name": "vsys_id", "type": "long"},
{"name": "rule_id", "type": "long"},
{"name": "template_id", "type": "long"},
{"name": "chart_id", "type": "long"},
{"name": "version", "type": "long"},
{"name": "client_ip_object_list", "type": "string", "multiValueHandling": "SORTED_SET"},
{"name": "server_ip_object_list", "type": "string", "multiValueHandling": "SORTED_SET"},
{"name": "fqdn_category_list", "type": "string", "multiValueHandling": "SORTED_SET"},
"client_ip",
"server_ip",
"client_country",
"server_country",
{"name": "client_asn", "type": "long"},
{"name": "server_asn", "type": "long"},
"server_fqdn",
"server_domain",
"app",
"app_category",
{"name": "c2s_ttl", "type": "long"},
{"name": "s2c_ttl", "type": "long"},
{"name": "c2s_link_id", "type": "long"},
{"name": "s2c_link_id", "type": "long"},
{"name": "client_port", "type": "long"},
{"name": "server_port", "type": "long"},
"ssl_ja3_hash",
"http_host",
"ssl_sni",
"subscriber_id",
"decoded_as",
"sled_ip",
{"name": "flags", "type": "long"},
"direction",
"bucket_id"
]
}
}
},
"metricsSpec": [
{"type": "longSum", "name": "count_1", "fieldName": "count_1"},
{"type": "longSum", "name": "count_2", "fieldName": "count_2"},
{"type": "longSum", "name": "count_3", "fieldName": "count_3"},
{"type": "longSum", "name": "count_4", "fieldName": "count_4"},
{"type": "HLLDSketchMerge", "name": "unique_count_1", "fieldName": "unique_count_1"},
{"type": "HLLDSketchMerge", "name": "unique_count_2", "fieldName": "unique_count_2"},
{"type": "HLLDSketchMerge", "name": "unique_count_3", "fieldName": "unique_count_3"},
{"type": "HLLDSketchMerge", "name": "unique_count_4", "fieldName": "unique_count_4"},
{"type": "HdrHistogramSketchMerge", "name": "distribution_1", "fieldName": "distribution_1"},
{"type": "HdrHistogramSketchMerge", "name": "distribution_2", "fieldName": "distribution_2"},
{"type": "HdrHistogramSketchMerge", "name": "distribution_3", "fieldName": "distribution_3"},
{"type": "HdrHistogramSketchMerge", "name": "distribution_4", "fieldName": "distribution_4"}
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec": {
"transforms": [
{"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
{"type": "expression", "name": "client_ip_object_list", "expression": "STRING_TO_ARRAY(client_ip_object_list,',')"},
{"type": "expression", "name": "server_ip_object_list", "expression": "STRING_TO_ARRAY(server_ip_object_list,',')"},
{"type": "expression", "name": "fqdn_category_list", "expression": "STRING_TO_ARRAY(fqdn_category_list,',')"},
{"type": "expression", "name": "count_1", "expression": "nvl(count_1, 0)"},
{"type": "expression", "name": "count_2", "expression": "nvl(count_2, 0)"},
{"type": "expression", "name": "count_3", "expression": "nvl(count_3, 0)"},
{"type": "expression", "name": "count_4", "expression": "nvl(count_4, 0)"},
{"type": "expression", "name": "bucket_id", "expression": "dimension_bucket(1024,device_id,rule_id,template_id,chart_id,version,client_ip_object_list,server_ip_object_list,fqdn_category_list,client_ip,server_ip,client_country,server_country,client_asn,server_asn,server_fqdn,server_domain,app,app_category,c2s_ttl,s2c_ttl,c2s_link_id,s2c_link_id,client_port,server_port,ssl_ja3_hash,http_host,ssl_sni,subscriber_id,decoded_as,sled_ip,flags,direction)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "statistics_rule"}
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"maxRowsInMemory": 250000,
"maxBytesInMemory": -1,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "STATISTICS-RULE-METRIC",
"taskCount": 2,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
},
"context":{
"druid.indexer.runner.javaOpts": "-server -Xms4g -Xmx4g -XX:MaxDirectMemorySize=2g"
}
}

104
druid/templates/tasks/index_kafka_statistics_rule_hits.json Normal file
View File

@@ -0,0 +1,104 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "statistics_rule_hits",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"device_group",
"data_center",
{
"name": "vsys_id",
"type": "long"
},
{
"name": "rule_id",
"type": "long"
}
]
}
}
},
"metricsSpec": [
{
"type": "longSum",
"name": "in_bytes",
"fieldName": "in_bytes"
},
{
"type": "longSum",
"name": "out_bytes",
"fieldName": "out_bytes"
},
{
"type": "longSum",
"name": "hit_count",
"fieldName": "hit_count"
},
{
"type": "longSum",
"name": "in_pkts",
"fieldName": "in_pkts"
},
{
"type": "longSum",
"name": "out_pkts",
"fieldName": "out_pkts"
}
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {
"type": "period",
"period": "PT1S"
},
"rollup": true
},
"transformSpec": {
"transforms": [
{"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {
"type": "selector",
"dimension": "name",
"value": "statistics_rule_hits"
}
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "POLICY-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

73
druid/templates/tasks/index_kafka_top_client_countries.json Normal file
View File

@@ -0,0 +1,73 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_client_countries",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
"rank_by",
"client_country"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "pkts", "fieldName" : "pkts" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "top_client_country"}
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOPK-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

73
druid/templates/tasks/index_kafka_top_client_ips.json Normal file
View File

@@ -0,0 +1,73 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_client_ips",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
"rank_by",
"client_ip"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "pkts", "fieldName" : "pkts" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "top_client_ip"}
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOPK-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

74
druid/templates/tasks/index_kafka_top_external_ips.json Normal file
View File

@@ -0,0 +1,74 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_external_ips",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
"rank_by",
"external_ip"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "pkts", "fieldName" : "pkts" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "top_external_ip"}
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOPK-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

74
druid/templates/tasks/index_kafka_top_internal_ips.json Normal file
View File

@@ -0,0 +1,74 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_internal_ips",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
"rank_by",
"internal_ip"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "pkts", "fieldName" : "pkts" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "top_internal_ip"}
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOPK-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

73
druid/templates/tasks/index_kafka_top_server_countries.json Normal file
View File

@@ -0,0 +1,73 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_server_countries",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
"rank_by",
"server_country"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "pkts", "fieldName" : "pkts" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "top_server_country"}
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOPK-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

74
druid/templates/tasks/index_kafka_top_server_domains.json Normal file
View File

@@ -0,0 +1,74 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_server_domains",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
"rank_by",
"server_domain"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "pkts", "fieldName" : "pkts" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "top_server_domain"}
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOPK-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

74
druid/templates/tasks/index_kafka_top_server_fqdns.json Normal file
View File

@@ -0,0 +1,74 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_server_fqdns",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
"rank_by",
"server_fqdn"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "pkts", "fieldName" : "pkts" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "top_server_fqdn"}
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOPK-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

74
druid/templates/tasks/index_kafka_top_server_ips.json Normal file
View File

@@ -0,0 +1,74 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_server_ips",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
"rank_by",
"server_ip"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "pkts", "fieldName" : "pkts" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "pkts", "expression": "nvl(pkts, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
],
"filter": {"type": "selector", "dimension": "name", "value": "top_server_ip"}
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOPK-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

83
druid/templates/tasks/index_kafka_traffic_general_stat.json Normal file
View File

@@ -0,0 +1,83 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "traffic_general_stat",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"device_group",
"data_center",
{ "name": "vsys_id", "type": "long" }
]
}
}
},
"metricsSpec":[
{ "type":"longSum", "name":"sessions", "fieldName":"sessions" },
{ "type":"longMax", "name":"active_sessions", "fieldName":"active_sessions" },
{ "type":"longSum", "name":"closed_sessions", "fieldName":"closed_sessions" },
{ "type":"longSum", "name":"in_bytes", "fieldName":"in_bytes" },
{ "type":"longSum", "name":"out_bytes", "fieldName":"out_bytes" },
{ "type":"longSum", "name":"in_pkts", "fieldName":"in_pkts" },
{ "type":"longSum", "name":"out_pkts", "fieldName":"out_pkts" },
{ "type":"longSum", "name":"asymmetric_c2s_flows", "fieldName":"asymmetric_c2s_flows" },
{ "type":"longSum", "name":"asymmetric_s2c_flows", "fieldName":"asymmetric_s2c_flows" },
{ "type":"longSum", "name":"tcp_syn_pkts", "fieldName":"tcp_syn_pkts" },
{ "type":"longSum", "name":"tcp_ack_pkts", "fieldName":"tcp_ack_pkts" },
{ "type":"longSum", "name":"tcp_fin_pkts", "fieldName":"tcp_fin_pkts" },
{ "type":"longSum", "name":"tcp_rst_pkts", "fieldName":"tcp_rst_pkts" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
{"type": "expression", "name": "active_sessions", "expression": "nvl(active_sessions, 0)"},
{"type": "expression", "name": "closed_sessions", "expression": "nvl(closed_sessions, 0)"},
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"},
{"type": "expression", "name": "asymmetric_c2s_flows", "expression": "nvl(asymmetric_c2s_flows, 0)"},
{"type": "expression", "name": "asymmetric_s2c_flows", "expression": "nvl(asymmetric_s2c_flows, 0)"},
{"type": "expression", "name": "tcp_syn_pkts", "expression": "nvl(tcp_syn_pkts, 0)"},
{"type": "expression", "name": "tcp_ack_pkts", "expression": "nvl(tcp_ack_pkts, 0)"},
{"type": "expression", "name": "tcp_fin_pkts", "expression": "nvl(tcp_fin_pkts, 0)"},
{"type": "expression", "name": "tcp_rst_pkts", "expression": "nvl(tcp_rst_pkts, 0)"}
],
"filter": { "type": "selector", "dimension": "name", "value": "traffic_general_stat" }
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "NETWORK-TRAFFIC-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

76
druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json Normal file
View File

@@ -0,0 +1,76 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "traffic_shaping_rule_hits",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp_ms",
"format": "millis"
},
"dimensionsSpec": {
"dimensions": [
{"name": "vsys_id","type": "long"},
"device_id",
"device_group",
"data_center",
{"name": "rule_id","type": "long"},
{"name": "profile_id","type": "long"},
{"name": "priority","type": "long"}
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
{ "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
{ "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" },
{ "type" : "longSum", "name" : "in_drop_pkts", "fieldName" : "in_drop_pkts" },
{ "type" : "longSum", "name" : "out_drop_pkts", "fieldName" : "out_drop_pkts" },
{"type": "HdrHistogramSketchMerge", "name": "latency_distribution_us", "fieldName": "latency_distribution_us"},
{ "type" : "longMax", "name" : "in_queue_len", "fieldName" : "in_queue_len" },
{ "type" : "longMax", "name" : "out_queue_len", "fieldName" : "out_queue_len" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1S"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
{"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
{"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
{"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"},
{"type": "expression", "name": "in_drop_pkts", "expression": "nvl(in_drop_pkts, 0)"},
{"type": "expression", "name": "out_drop_pkts", "expression": "nvl(out_drop_pkts, 0)"},
{"type": "expression", "name": "in_queue_len", "expression": "nvl(in_queue_len, 0)"},
{"type": "expression", "name": "out_queue_len", "expression": "nvl(out_queue_len, 0)"}
],
"filter": { "type": "selector", "dimension": "name", "value": "traffic_shaping_rule_hits" }
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "POLICY-RULE-METRIC",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}