gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交clickhouse和druid更新文件

Browse Source
This commit is contained in:
qidaijie
2021-09-09 17:13:01 +08:00
parent 20ab6fbf95
commit a9366e264e
22 changed files with 3917 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
TSG发布版本更新记录/TSG-21.09/Clickhouse/bloom_filter.sql Normal file
View File

@@ -0,0 +1,3 @@
alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1;
alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1;
alter table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1;

2776
TSG发布版本更新记录/TSG-21.09/Clickhouse/rename.sql Normal file
View File

File diff suppressed because it is too large Load Diff

17
TSG发布版本更新记录/TSG-21.09/Druid/tasks.txt Normal file
View File

@@ -0,0 +1,17 @@
proxy_event_hits_log.json
security_event_hits_log.json
sys_storage_log.json
top_client_ip_log.json
top_external_host_log.json
top_internal_host_log.json
top_server_ip_log.json
top_urls_log.json
top_user_log.json
top_website_domain_log.json
traffic_metrics_log.json
traffic_protocol_stat_log.json
traffic_summary_log.json
traffic_top_destination_ip_metrics_log.json
urls_proxy_hot.json
urls_security_hot.json
traffic_app_stat_log.json

96
TSG发布版本更新记录/TSG-21.09/Druid/tasks/proxy_event_hits_log.json Normal file
View File

@@ -0,0 +1,96 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "proxy_event_hits_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "common_recv_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
{ "name": "policy_id", "type": "long" },
"isp",
"country",
"location",
{ "name": "entrance_id", "type": "long" },
{ "name": "action", "type": "long" },
"sub_action"
]
}
}
},
"metricsSpec" : [
{ "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" },
{ "type" : "count", "name" : "hits" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{
"type": "expression",
"name": "country",
"expression": "array_ordinal(string_to_array(common_client_location,','),3)"
},
{
"type": "expression",
"name": "location",
"expression": "array_ordinal(string_to_array(common_client_location,','),2)"
},
{
"type": "expression",
"name": "policy_id",
"expression": "common_policy_id"
},
{
"type": "expression",
"name": "isp",
"expression": "common_isp"
},
{
"type": "expression",
"name": "entrance_id",
"expression": "common_entrance_id"
},
{
"type": "expression",
"name": "sub_action",
"expression": "common_sub_action"
},
{
"type": "expression",
"name": "action",
"expression": "common_action"
}
]
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 20000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "PROXY-EVENT-COMPLETED",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

76
TSG发布版本更新记录/TSG-21.09/Druid/tasks/security_event_hits_log.json Normal file
View File

@@ -0,0 +1,76 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "security_event_hits_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "common_recv_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
{ "name": "policy_id", "type": "long" },
"isp",
{ "name": "entrance_id", "type": "long" },
{ "name": "action", "type": "long" }
]
}
}
},
"metricsSpec" : [
{ "type" : "count", "name" : "hits" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT1M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{
"type": "expression",
"name": "policy_id",
"expression": "common_policy_id"
},
{
"type": "expression",
"name": "isp",
"expression": "common_isp"
},
{
"type": "expression",
"name": "entrance_id",
"expression": "common_entrance_id"
},
{
"type": "expression",
"name": "action",
"expression": "common_action"
}
]
}
},
"tuningConfig": {
"type": "kafka",
"reportParseExceptions": false
},
"ioConfig": {
"topic": "SECURITY-EVENT-COMPLETED",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

51
TSG发布版本更新记录/TSG-21.09/Druid/tasks/sys_storage_log.json Normal file
View File

@@ -0,0 +1,51 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "sys_storage_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"log_type",
"data_center",
{ "name": "used_size", "type": "long" },
{ "name": "max_size", "type": "long" },
{ "name": "aggregate_size", "type": "long" },
{ "name": "last_storage", "type": "long" }
]
}
}
},
"metricsSpec" : [
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": false
}
},
"tuningConfig": {
"type": "kafka",
"reportParseExceptions": false
},
"ioConfig": {
"topic": "SYS-STORAGE",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

52
TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_client_ip_log.json Normal file
View File

@@ -0,0 +1,52 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_client_ip_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "stat_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
"source",
"order_by"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" },
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
}
},
"tuningConfig": {
"type": "kafka",
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOP-CLIENT-IP",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

53
TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_external_host_log.json Normal file
View File

@@ -0,0 +1,53 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_external_host_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "stat_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
"destination",
"order_by"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" },
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOP-EXTERNAL-HOST",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

52
TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_internal_host_log.json Normal file
View File

@@ -0,0 +1,52 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_internal_host_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "stat_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
"source",
"order_by"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" },
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
}
},
"tuningConfig": {
"type": "kafka",
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOP-INTERNAL-HOST",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

53
TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_server_ip_log.json Normal file
View File

@@ -0,0 +1,53 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_server_ip_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "stat_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
"destination",
"order_by"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" },
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOP-SERVER-IP",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

49
TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_urls_log.json Normal file
View File

@@ -0,0 +1,49 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_urls_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "stat_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
"url"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOP-URLS",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

52
TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_user_log.json Normal file
View File

@@ -0,0 +1,52 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_user_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "stat_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
"subscriber_id",
"order_by"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" },
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
}
},
"tuningConfig": {
"type": "kafka",
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOP-USER",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

53
TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_website_domain_log.json Normal file
View File

@@ -0,0 +1,53 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "top_website_domain_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "stat_time",
"format": "iso"
},
"dimensionsSpec": {
"dimensions": [
"domain",
"order_by"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" },
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TOP-WEBSITE-DOMAIN",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

63
TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_app_stat_log.json Normal file
View File

@@ -0,0 +1,63 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "traffic_app_stat_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "common_recv_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"app_name",
"sub_app_name"
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_num","fieldName" : "common_sessions"},
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression", "name": "app_name", "expression": "array_ordinal(string_to_array(replace(common_app_label, '.', '/'),'/'),1)"},
{"type": "expression", "name": "sub_app_name", "expression": "array_ordinal(string_to_array(replace(common_app_label, '.', '/'),'/'),2)"}
],
"filter": {
"type": "not",
"field":{ "type": "selector", "dimension": "app_name", "value": "" }
}
}
},
"tuningConfig": {
"type": "kafka",
"reportParseExceptions": false
},
"ioConfig": {
"topic": "SESSION-RECORD-COMPLETED",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"earlyMessageRejectionPeriod": "PT6H",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

152
TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_metrics_log.json Normal file
View File

@@ -0,0 +1,152 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "traffic_metrics_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "timestamp",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"device_id",
"entrance_id"
]
},
"flattenSpec": {
"useFieldDiscovery": true,
"fields": [
{ "name": "allow_conn_num", "type": "path", "expr": "$.fields.allow_conn_num" },
{ "name": "allow_in_bytes", "type": "path", "expr": "$.fields.allow_in_bytes" },
{ "name": "allow_in_packets", "type": "path", "expr": "$.fields.allow_in_packets" },
{ "name": "allow_out_bytes", "type": "path", "expr": "$.fields.allow_out_bytes" },
{ "name": "allow_out_packets", "type": "path", "expr": "$.fields.allow_out_packets" },
{ "name": "close_conn_num", "type": "path", "expr": "$.fields.close_conn_num" },
{ "name": "default_conn_num", "type": "path", "expr": "$.fields.default_conn_num" },
{ "name": "default_in_bytes", "type": "path", "expr": "$.fields.default_in_bytes" },
{ "name": "default_in_packets", "type": "path", "expr": "$.fields.default_in_packets" },
{ "name": "default_out_bytes", "type": "path", "expr": "$.fields.default_out_bytes" },
{ "name": "default_out_packets", "type": "path", "expr": "$.fields.default_out_packets" },
{ "name": "deny_conn_num", "type": "path", "expr": "$.fields.deny_conn_num" },
{ "name": "deny_in_bytes", "type": "path", "expr": "$.fields.deny_in_bytes" },
{ "name": "deny_in_packets", "type": "path", "expr": "$.fields.deny_in_packets" },
{ "name": "deny_out_bytes", "type": "path", "expr": "$.fields.deny_out_bytes" },
{ "name": "deny_out_packets", "type": "path", "expr": "$.fields.deny_out_packets" },
{ "name": "established_conn_num", "type": "path", "expr": "$.fields.established_conn_num" },
{ "name": "intercept_conn_num", "type": "path", "expr": "$.fields.intercept_conn_num" },
{ "name": "intercept_in_bytes", "type": "path", "expr": "$.fields.intercept_in_bytes" },
{ "name": "intercept_in_packets", "type": "path", "expr": "$.fields.intercept_in_packets" },
{ "name": "intercept_out_bytes", "type": "path", "expr": "$.fields.intercept_out_bytes" },
{ "name": "intercept_out_packets", "type": "path", "expr": "$.fields.intercept_out_packets" },
{ "name": "monitor_conn_num", "type": "path", "expr": "$.fields.monitor_conn_num" },
{ "name": "monitor_in_bytes", "type": "path", "expr": "$.fields.monitor_in_bytes" },
{ "name": "monitor_in_packets", "type": "path", "expr": "$.fields.monitor_in_packets" },
{ "name": "monitor_out_bytes", "type": "path", "expr": "$.fields.monitor_out_bytes" },
{ "name": "monitor_out_packets", "type": "path", "expr": "$.fields.monitor_out_packets" },
{ "name": "new_conn_num", "type": "path", "expr": "$.fields.new_conn_num" },
{ "name": "total_in_bytes", "type": "path", "expr": "$.fields.total_in_bytes" },
{ "name": "total_in_packets", "type": "path", "expr": "$.fields.total_in_packets" },
{ "name": "total_out_bytes", "type": "path", "expr": "$.fields.total_out_bytes" },
{ "name": "total_out_packets", "type": "path", "expr": "$.fields.total_out_packets" },
{ "name": "pinning_num", "type": "path", "expr": "$.fields.pinning_num" },
{ "name": "not_pinning_num", "type": "path", "expr": "$.fields.not_pinning_num" },
{ "name": "maybe_pinning_num", "type": "path", "expr": "$.fields.maybe_pinning_num" },
{ "name": "alert_bytes", "type": "path", "expr": "$.fields.alert_bytes" },
{ "name": "insert_bytes", "type": "path", "expr": "$.fields.insert_bytes" },
{ "name": "hijack_bytes", "type": "path", "expr": "$.fields.hijack_bytes" },
{ "name": "ad_reflection_bytes", "type": "path", "expr": "$.fields.ad_reflection_bytes" },
{ "name": "ad_flood_bytes", "type": "path", "expr": "$.fields.ad_flood_bytes" },
{ "name": "ad_cc_bytes", "type": "path", "expr": "$.fields.ad_cc_bytes" },
{ "name": "intercept_monitor_conn_num", "type": "path", "expr": "$.fields.intcp_mon_num" },
{ "name": "intercept_allow_conn_num", "type": "path", "expr": "$.fields.intcp_allow_num" },
{ "name": "intercept_deny_conn_num", "type": "path", "expr": "$.fields.intcp_deny_num" },
{ "name": "intercept_redirect_conn_num", "type": "path", "expr": "$.fields.intcp_rdirt_num" },
{ "name": "intercept_replace_conn_num", "type": "path", "expr": "$.fields.intcp_repl_num" },
{ "name": "intercept_hijack_conn_num", "type": "path", "expr": "$.fields.intcp_hijk_num" },
{ "name": "intercept_insert_conn_num", "type": "path", "expr": "$.fields.intcp_ins_num" },
{ "name": "app_name", "type": "path", "expr": "$.tags.app_name" },
{ "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
{ "name": "entrance_id", "type": "path", "expr": "$.tags.entrance_id" }
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "new_conn_num", "fieldName" : "new_conn_num" },
{ "type" : "longMax", "name" : "established_conn_num", "fieldName" : "established_conn_num" },
{ "type" : "longSum", "name" : "close_conn_num", "fieldName" : "close_conn_num" },
{ "type" : "longSum", "name" : "total_in_bytes", "fieldName" : "total_in_bytes" },
{ "type" : "longSum", "name" : "total_out_bytes", "fieldName" : "total_out_bytes" },
{ "type" : "longSum", "name" : "total_in_packets", "fieldName" : "total_in_packets" },
{ "type" : "longSum", "name" : "total_out_packets", "fieldName" : "total_out_packets" },
{ "type" : "longSum", "name" : "default_conn_num", "fieldName" : "default_conn_num" },
{ "type" : "longSum", "name" : "default_in_bytes", "fieldName" : "default_in_bytes" },
{ "type" : "longSum", "name" : "default_out_bytes", "fieldName" : "default_out_bytes" },
{ "type" : "longSum", "name" : "default_in_packets", "fieldName" : "default_in_packets" },
{ "type" : "longSum", "name" : "default_out_packets", "fieldName" : "default_out_packets" },
{ "type" : "longSum", "name" : "allow_conn_num", "fieldName" : "allow_conn_num" },
{ "type" : "longSum", "name" : "allow_in_bytes", "fieldName" : "allow_in_bytes" },
{ "type" : "longSum", "name" : "allow_out_bytes", "fieldName" : "allow_out_bytes" },
{ "type" : "longSum", "name" : "allow_in_packets", "fieldName" : "allow_in_packets" },
{ "type" : "longSum", "name" : "allow_out_packets", "fieldName" : "allow_out_packets" },
{ "type" : "longSum", "name" : "deny_conn_num", "fieldName" : "deny_conn_num" },
{ "type" : "longSum", "name" : "deny_in_bytes", "fieldName" : "deny_in_bytes" },
{ "type" : "longSum", "name" : "deny_out_bytes", "fieldName" : "deny_out_bytes" },
{ "type" : "longSum", "name" : "deny_in_packets", "fieldName" : "deny_in_packets" },
{ "type" : "longSum", "name" : "deny_out_packets", "fieldName" : "deny_out_packets" },
{ "type" : "longSum", "name" : "monitor_conn_num", "fieldName" : "monitor_conn_num" },
{ "type" : "longSum", "name" : "monitor_in_bytes", "fieldName" : "monitor_in_bytes" },
{ "type" : "longSum", "name" : "monitor_out_bytes", "fieldName" : "monitor_out_bytes" },
{ "type" : "longSum", "name" : "monitor_in_packets", "fieldName" : "monitor_in_packets" },
{ "type" : "longSum", "name" : "monitor_out_packets", "fieldName" : "monitor_out_packets" },
{ "type" : "longSum", "name" : "intercept_conn_num", "fieldName" : "intercept_conn_num" },
{ "type" : "longSum", "name" : "intercept_in_bytes", "fieldName" : "intercept_in_bytes" },
{ "type" : "longSum", "name" : "intercept_out_bytes", "fieldName" : "intercept_out_bytes" },
{ "type" : "longSum", "name" : "intercept_in_packets", "fieldName" : "intercept_in_packets" },
{ "type" : "longSum", "name" : "intercept_out_packets", "fieldName" : "intercept_out_packets" },
{ "type" : "longSum", "name" : "pinning_num", "fieldName" : "pinning_num" },
{ "type" : "longSum", "name" : "not_pinning_num", "fieldName" : "not_pinning_num" },
{ "type" : "longSum", "name" : "maybe_pinning_num", "fieldName" : "maybe_pinning_num" },
{ "type" : "longSum", "name" : "alert_bytes", "fieldName" : "alert_bytes" },
{ "type" : "longSum", "name" : "ins_bytes", "fieldName" : "insert_bytes" },
{ "type" : "longSum", "name" : "hijk_bytes", "fieldName" : "hijack_bytes" },
{ "type" : "longSum", "name" : "ad_reflection_bytes", "fieldName" : "ad_reflection_bytes" },
{ "type" : "longSum", "name" : "ad_flood_bytes", "fieldName" : "ad_flood_bytes" },
{ "type" : "longSum", "name" : "ad_cc_bytes", "fieldName" : "ad_cc_bytes" },
{ "type" : "longSum", "name" : "intcp_mon_num", "fieldName" : "intercept_monitor_conn_num" },
{ "type" : "longSum", "name" : "intcp_allow_num", "fieldName" : "intercept_allow_conn_num" },
{ "type" : "longSum", "name" : "intcp_deny_num", "fieldName" : "intercept_deny_conn_num" },
{ "type" : "longSum", "name" : "intcp_rdirt_num", "fieldName" : "intercept_redirect_conn_num" },
{ "type" : "longSum", "name" : "intcp_repl_num", "fieldName" : "intercept_replace_conn_num" },
{ "type" : "longSum", "name" : "intcp_hijk_num", "fieldName" : "intercept_hijack_conn_num" },
{ "type" : "longSum", "name" : "intcp_ins_num", "fieldName" : "intercept_insert_conn_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5S", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{}
},
"tuningConfig": {
"type": "kafka",
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TRAFFIC-METRICS",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

65
TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_protocol_stat_log.json Normal file
View File

@@ -0,0 +1,65 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "traffic_protocol_stat_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "stat_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"protocol_id",
"isp",
"data_center",
{
"name": "entrance_id",
"type": "long"
}
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" },
{ "type" : "longSum", "name" : "c2s_ipfrag_num", "fieldName" : "c2s_ipfrag_num" },
{ "type" : "longSum", "name" : "s2c_ipfrag_num", "fieldName" : "s2c_ipfrag_num" },
{ "type" : "longSum", "name" : "c2s_tcp_lostlen", "fieldName" : "c2s_tcp_lostlen" },
{ "type" : "longSum", "name" : "s2c_tcp_lostlen", "fieldName" : "s2c_tcp_lostlen" },
{ "type" : "longSum", "name" : "c2s_tcp_unorder_num", "fieldName" : "c2s_tcp_unorder_num" },
{ "type" : "longSum", "name" : "s2c_tcp_unorder_num", "fieldName" : "s2c_tcp_unorder_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT15S", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TRAFFIC-PROTOCOL-STAT",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

74
TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_summary_log.json Normal file
View File

@@ -0,0 +1,74 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "traffic_summary_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "common_recv_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"isp",
"data_center",
"schema_type",
{ "name": "entrance_id", "type": "long" }
]
}
}
},
"metricsSpec" : [
{ "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" },
{ "type" : "longSum", "name" : "sessions", "fieldName" : "common_sessions" },
{ "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" },
{ "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" },
{ "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" },
{ "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" },
{ "type" : "longSum", "name" : "one_sided_connections", "fieldName" : "one_sided_connections" },
{ "type" : "longSum", "name" : "uncategorized_bytes", "fieldName" : "uncategorized_bytes" },
{ "type" : "longSum", "name" : "fragmentation_packets", "fieldName" : "fragmentation_packets" },
{ "type" : "longSum", "name" : "sequence_gap_loss", "fieldName" : "sequence_gap_loss" },
{ "type" : "longSum", "name" : "unorder_packets", "fieldName" : "unorder_packets" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT15S", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{"type": "expression","name": "isp","expression": "common_isp"},
{"type": "expression","name": "data_center","expression": "common_data_center"},
{"type": "expression","name": "entrance_id","expression": "common_entrance_id"},
{"type": "expression","name": "schema_type","expression": "common_schema_type"},
{"type": "expression", "name": "one_sided_connections", "expression": "if(common_stream_dir!=3,common_sessions,0)"},
{"type": "expression", "name": "uncategorized_bytes", "expression": "if(array_ordinal(string_to_array(common_protocol_label,'/'),1)=='UNCATEGORIZED',\"common_c2s_byte_num\"+\"common_s2c_byte_num\",0)"},
{"type": "expression","name": "fragmentation_packets","expression": "\"common_c2s_ipfrag_num\"+\"common_s2c_ipfrag_num\""},
{"type": "expression","name": "sequence_gap_loss","expression": "\"common_c2s_tcp_lostlen\"+\"common_s2c_tcp_lostlen\""},
{"type": "expression","name": "unorder_packets","expression": "\"common_c2s_tcp_unorder_num\"+\"common_s2c_tcp_unorder_num\""}
]
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 20000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "SESSION-RECORD-COMPLETED",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

58
TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_top_destination_ip_metrics_log.json Normal file
View File

@@ -0,0 +1,58 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "traffic_top_destination_ip_metrics_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "sketch_start_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"common_sled_ip",
"destination_ip",
"common_data_center",
"attack_type",
{ "type": "long", "name": "partition_num" }
]
}
}
},
"metricsSpec" : [
{ "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"},
{ "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" },
{ "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" },
{ "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT10M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{
"transforms":[]
}
},
"tuningConfig": {
"type": "kafka",
"resetOffsetAutomatically": true,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "TRAFFIC-TOP-DESTINATION-IP-METRICS",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

61
TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_proxy_hot.json Normal file
View File

@@ -0,0 +1,61 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "proxy_event_urls_hot_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "common_recv_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"url"
]
}
}
},
"metricsSpec" : [
{ "type" : "count", "name" : "session_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{
"type": "expression",
"name": "url",
"expression": "http_url"
}
],
"filter": {
"type": "not",
"field":{ "type": "selector", "dimension": "url", "value": "" }
}
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "PROXY-EVENT-COMPLETED",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

61
TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_security_hot.json Normal file
View File

@@ -0,0 +1,61 @@
{
"type": "kafka",
"dataSchema": {
"dataSource": "security_event_urls_hot_log",
"parser": {
"type": "string",
"parseSpec": {
"format": "json",
"timestampSpec": {
"column": "common_recv_time",
"format": "posix"
},
"dimensionsSpec": {
"dimensions": [
"url"
]
}
}
},
"metricsSpec" : [
{ "type" : "count", "name" : "session_num" }
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "DAY",
"queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
"rollup": true
},
"transformSpec" :{
"transforms":[
{
"type": "expression",
"name": "url",
"expression": "http_url"
}
],
"filter": {
"type": "not",
"field":{ "type": "selector", "dimension": "url", "value": "" }
}
}
},
"tuningConfig": {
"type": "kafka",
"maxRowsPerSegment": 5000000,
"reportParseExceptions": false
},
"ioConfig": {
"topic": "SECURITY-EVENT-COMPLETED",
"taskCount": 1,
"replicas": 1,
"taskDuration": "PT1H",
"completionTimeout": "PT30M",
"consumerProperties": {
"bootstrap.servers": "kafkabootstrap",
"sasl.mechanism": "PLAIN",
"security.protocol": "SASL_PLAINTEXT",
"sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
}
}
}

0
TSG发布版本更新记录/TSG-21.09/Clickhouse/.gitkeep → TSG发布版本更新记录/TSG-21.09/Flink/.gitkeep
View File

0
TSG发布版本更新记录/TSG-21.09/Druid/.gitkeep → TSG发布版本更新记录/TSG-21.09/Gohangout/.gitkeep
View File