gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

21.11 update

Browse Source
This commit is contained in:
zhanghongqing
2021-11-22 17:01:55 +08:00
parent b22055b8ae
commit 091c7346c6
59 changed files with 19687 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
TSG发布版本更新记录/TSG-21.09/dos-detection/.gitkeep
View File

0
TSG发布版本更新记录/TSG-21.11/.gitkeep
View File

39
TSG发布版本更新记录/TSG-21.11/flink/flink-conf.yml Normal file
View File

@@ -0,0 +1,39 @@
# flink集群版 配置文件增加checkpoint history
#==============================================================================
#checkpoint
#==============================================================================
#checkpoint路径
state.checkpoints.dir: hdfs:///flink/checkpoints
#检查点间隔
#execution.checkpointing.interval: 300000
execution.checkpointing.interval: 300000
#检查点的模式
execution.checkpointing.mode: EXACTLY_ONCE
#检查点的超时设置
execution.checkpointing.timeout: 10min
#检查点的个数
state.checkpoints.num-retained: 20
state.backend: filesystem
#==============================================================================
#historyserver
#==============================================================================
#完成的作业到期并从作业存储区中清除的秒数
jobstore.expiration-time: 7200
#flink历史任务
# 上传已完成作业信息的目录
jobmanager.archive.fs.dir: hdfs:///history-jobs
# 监视以下目录中已完成的作业
historyserver.archive.fs.dir: hdfs:///history-jobs
#
# # 每 10 秒刷新一次
historyserver.archive.fs.refresh-interval: 10000
#避免由于task不能正常取消而使taskmanager服务挂掉
task.cancellation.timeout: 0

8
TSG发布版本更新记录/TSG-21.11/hbase/create_table
View File

@@ -1,9 +1,11 @@
create_namespace 'tsg_galaxy'
create 'tsg_galaxy:relation_account_framedip', {NAME => 'radius', VERSIONS => 1}
create 'tsg_galaxy:relation_framedip_account', {NAME => 'radius', VERSIONS => 1}
-- Phoenix创建视图
CREATE view "tsg"."report_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR);
CREATE view "tsg_galaxy"."relation_account_framedip" ( ROWKEY VARCHAR PRIMARY KEY, "radius"."framed_ip" VARCHAR, "radius"."last_found_time" UNSIGNED_LONG);
CREATE view "tsg_galaxy"."relation_account_framedip" ( ROWKEY VARCHAR PRIMARY KEY, "radius"."framed_ip" VARCHAR, "radius"."last_found_time" UNSIGNED_LONG);

12
TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml Normal file
View File

@@ -0,0 +1,12 @@
<!-- hbase hbase-site.xml 新增以下配置,集成Phoenix -->
<configuration>
<property>
<name>phoenix.schema.isNamespaceMappingEnabled</name>
<value>true</value>
</property>
<property>
<name>phoenix.schema.mapSystemTablesToNamespace</name>
<value>true</value>
</property>
</configuration>

3
TSG发布版本更新记录/TSG-21.11/job/xxl_job.sql Normal file
View File

@@ -0,0 +1,3 @@
-- 优化了druid压缩任务，需要先删除xxl_job_info 中的任务
INSERT INTO `xxl_job_info` VALUES ('86', '6', '0 0 1 * * ?', 'Compact-With-Day-Job', '2021-07-21 14:13:57', '2021-11-22 06:56:01', 'galaxy', '', 'FAILOVER', 'druidCompactSegmentJobHandler', '{\r\n\"dataSource\":\"top_urls_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_website_domain_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_client_ip_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_server_ip_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_external_host_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_internal_host_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_user_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"security_event_hits_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"proxy_event_hits_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_protocol_stat_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_metrics_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_summary_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_top_destination_ip_metrics_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_app_stat_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"sys_storage_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2021-07-21 14:13:57', '', '1', '1637514000000', '1637600400000');

40
TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf
View File

@@ -1,28 +1,28 @@
worker_processes auto;
events {
worker_connections 4096;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
keepalive_requests 10000;
vhost_traffic_status_zone;
vhost_traffic_status_zone;
vhost_traffic_status_filter_by_host on;
#
client_max_body_size 1024M; #(设置客户端请求体最大值)
client_body_buffer_size 128k; #(配置请求体缓存区大小,)
client_max_body_size 1024M;
client_body_buffer_size 128k;
access_log off; # 关闭access日志
upstream qgwService {
server 192.168.44.10:8183;
server 192.168.44.13:8183;
}
upstream jobAdmin {
server 192.168.44.10:8184;
server 192.168.44.13:8184;
@@ -31,6 +31,13 @@ http {
upstream hos {
server 192.168.44.10:8186;
server 192.168.44.13:8186;
keepalive 1000;
keepalive_timeout 65;
keepalive_requests 100000;
}
geo $islocalip {
default 0;
#192.168.40.0/24 1; #添加本地不需要加密的ip，支持掩码形式，格式为xxx.xxx.xxx.xxx 1;
}
upstream nacos {
server 192.168.44.11:8848;
@@ -52,7 +59,7 @@ http {
gzip_vary on; #是否传输gzip压缩标志
}
}
server {
listen 8181;
server_name localhost;
@@ -67,17 +74,26 @@ http {
vhost_traffic_status_display;
vhost_traffic_status_display_format html;
}
}
#hos非加密
server {
listen 9098;
server_name localhost;
proxy_set_header Host $http_host;
client_max_body_size 1024m;
location / {
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $http_host;
set $ssl 1; #是否开启重定向加密，开启是1，关闭是0，默认关闭
if ($islocalip = 0){
set $ssl "${ssl}1";
}
if ($request_method = GET) {
set $ssl "${ssl}1";
}
if ($ssl = "111") {
return 302 https://$host:9097$request_uri;
}
proxy_pass http://hos$request_uri;

18
TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml Normal file
View File

@@ -0,0 +1,18 @@
# qgw 修改以下配置
spring:
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://{{ machine_host }}:3306/xxl_job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=GMT
username: {{ galaxy_mariadb_username }}
pin: {{ galaxy_mariadb_key }}
hbase:
zookeeperQuorum: {{ machine_host }}:2181
zookeeperZnodeParent: /hbase
rpcTimeout: 60000
report:
dbname: tsg
tableName: report_result
columnFamily: response
columnName: result

322
TSG发布版本更新记录/TSG-21.11/qgw/schema/active_defence_event.json Normal file
View File

@@ -0,0 +1,322 @@
{
"type": "record",
"name": "active_defence_event",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"partition_key": "common_recv_time",
"schema_query": {
"dimensions": [
"common_policy_id",
"ad_target_ip",
"ad_cc_target_url"
],
"metrics": [
"ad_target_ip",
"ad_sent_byte_num",
"ad_sent_pkt_num",
"ad_cc_initiate_connection_num",
"ad_cc_established_connection_num",
"ad_cc_rejected_connection_num"
],
"filters": [
"common_policy_id",
"ad_target_ip",
"ad_target_port",
"ad_protocol",
"common_address_type",
"ad_sent_byte_num",
"ad_sent_pkt_num",
"ad_cc_initiate_connection_num",
"ad_cc_established_connection_num",
"ad_cc_rejected_connection_num"
]
},
"schema_type": {
"REFLECTION": {
"columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"common_address_type",
"common_device_id",
"common_egress_link_id",
"common_ingress_link_id",
"common_entrance_id",
"common_user_region",
"ad_method",
"ad_protocol",
"ad_target_ip",
"ad_target_port",
"ad_target_ip_location",
"ad_target_ip_asn",
"ad_reflector_profile_id",
"ad_sent_pkt_num",
"ad_sent_byte_num",
"ad_generate_time"
],
"default_columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"ad_target_ip",
"ad_target_port",
"ad_reflector_profile_id",
"ad_sent_pkt_num",
"ad_sent_byte_num"
]
},
"FLOOD": {
"columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"common_address_type",
"common_device_id",
"common_egress_link_id",
"common_ingress_link_id",
"common_entrance_id",
"common_user_region",
"ad_method",
"ad_protocol",
"ad_target_ip",
"ad_target_port",
"ad_target_ip_location",
"ad_target_ip_asn",
"ad_claimed_src_ip_profile_id",
"ad_sent_pkt_num",
"ad_sent_byte_num",
"ad_generate_time"
],
"default_columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"ad_target_ip",
"ad_target_port",
"ad_claimed_src_ip_profile_id",
"ad_protocol"
]
},
"CC": {
"columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"common_address_type",
"common_device_id",
"common_egress_link_id",
"common_ingress_link_id",
"common_entrance_id",
"common_user_region",
"ad_method",
"ad_protocol",
"ad_cc_target_url",
"ad_claimed_src_ip_profile_id",
"ad_cc_initiate_connection_num",
"ad_cc_established_connection_num",
"ad_cc_rejected_connection_num",
"ad_generate_time"
],
"default_columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"ad_cc_target_url",
"ad_claimed_src_ip_profile_id",
"ad_protocol"
]
}
},
"default_columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"ad_target_ip",
"ad_target_port",
"ad_cc_target_url"
]
},
"fields": [
{
"name": "common_recv_time",
"label": "Receive Time",
"doc": {
"allow_query": "true",
"constraints": {
"type": "timestamp"
},
"format": {
"functions": "current_timestamp"
}
},
"type": "long"
},
{
"name": "common_log_id",
"label": "Log ID",
"doc": {
"allow_query": "true",
"format": {
"functions": "snowflake_id"
}
},
"type": "long"
},
{
"name": "common_policy_id",
"label": "Policy ID",
"doc": {
"allow_query": "true"
},
"type": "long"
},
{
"name": "common_address_type",
"label": "Address Type",
"doc": {
"data": [
{
"code": "4",
"value": "ipv4"
},
{
"code": "6",
"value": "ipv6"
}
]
},
"type": "int"
},
{
"name": "common_entrance_id",
"label": "Entrance ID",
"doc": {
"visibility": "disabled"
},
"type": "int"
},
{
"name": "common_device_id",
"label": "Device ID",
"type": "string"
},
{
"name": "common_egress_link_id",
"label": "Egress Link ID",
"doc": {
"visibility": "hidden"
},
"type": "int"
},
{
"name": "common_ingress_link_id",
"label": "Ingress Link ID",
"doc": {
"visibility": "hidden"
},
"type": "int"
},
{
"name": "common_user_region",
"label": "User Region",
"doc": {
"visibility": "hidden"
},
"type": "string"
},
{
"name": "ad_target_ip",
"label": "Target IP",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
},
"format": {
"functions": "geo_ip_country,geo_asn",
"appendTo": "ad_target_ip_location,ad_target_ip_asn"
}
},
"type": "string"
},
{
"name": "ad_target_port",
"label": "Target Port",
"type": "int"
},
{
"name": "ad_cc_target_url",
"label": "Target URL",
"doc": {
"allow_query": "true"
},
"type": "string"
},
{
"name": "ad_target_ip_location",
"label": "Target Location",
"type": "string"
},
{
"name": "ad_target_ip_asn",
"label": "Target ASN",
"type": "string"
},
{
"name": "ad_protocol",
"label": "Protocol",
"type": "string"
},
{
"name": "ad_method",
"label": "Method",
"type": "string"
},
{
"name": "ad_claimed_src_ip_profile_id",
"label": "Claimed Profile ID",
"type": "int"
},
{
"name": "ad_reflector_profile_id",
"label": "Reflector Profile ID",
"type": "int"
},
{
"name": "ad_sent_pkt_num",
"label": "Packets Sent",
"type": "int"
},
{
"name": "ad_sent_byte_num",
"label": "Bytes Sent",
"type": "int"
},
{
"name": "ad_cc_initiate_connection_num",
"label": "Initiate Numbers",
"type": "int"
},
{
"name": "ad_cc_established_connection_num",
"label": "Established Numbers",
"type": "int"
},
{
"name": "ad_cc_rejected_connection_num",
"label": "Rejected Numbers",
"type": "int"
},
{
"name": "ad_generate_time",
"label": "Generate Time",
"doc": {
"constraints": {
"type": "timestamp"
}
},
"type": "int"
}
]
}

99
TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-filter.json Normal file
View File

@@ -0,0 +1,99 @@
{
"version": "1.0",
"name": "ClickHouse-Raw",
"namespace": "ClickHouse",
"filters": [
{
"name":"@start",
"value": "'2021-10-19 10:00:00'"
},
{
"name":"@end",
"value": "'2021-10-20 11:00:00'"
},
{
"name":"@common_filter",
"value": [
"common_log_id=1153021139190754263",
"common_client_ip='118.180.48.74'",
"common_client_ip='120.242.132.200'",
"common_internal_ip='223.116.37.192'",
"common_server_ip='8.8.8.8'",
"common_server_ip='114.114.114.114'",
"common_server_ip!='114.114.114.114'",
"common_server_ip='120.239.72.226'",
"common_external_ip='111.10.53.14'",
"common_client_port=52607",
"common_server_port=443",
"common_c2s_pkt_num>5",
"common_s2c_pkt_num>5",
"common_c2s_byte_num>100",
"common_s2c_byte_num<200",
"common_schema_type='DNS'",
"common_establish_latency_ms>200",
"common_con_duration_ms>10000",
"common_stream_trace_id=1153021139190754263",
"common_tcp_client_isn=2857077935",
"common_tcp_server_isn=0",
"http_domain='qq.com'",
"http_domain!='qq.com'",
"http_domain='yunser.com'",
"mail_account='abc@xx.com'",
"mail_subject='test'",
"dns_qname='qbwup.imtt.qq.com'",
"ssl_sni='mmbiz.qpic.cn'",
"ssl_sni='openai.qq.com'",
"ssl_con_latency_ms>100",
"ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'",
"common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'",
"common_server_ip='111.10.53.14' and common_server_port=443",
"common_server_ip like '120.239%'",
"common_server_ip not like '120.239%'",
"common_server_ip like '%114.114%'",
"mail_account like 'abc@%'",
"http_domain like '%baidu.com%'",
"ssl_sni like '%google.com'",
"http_domain like 'baidu%'",
"http_domain like '%baidu.com%'",
"common_client_ip in ('120.239.72.226','114.114.114.114')",
"common_client_ip not in ('120.239.72.226','114.114.114.114')",
"common_server_ip='116.177.248.126' and notEmpty(http_domain)",
"common_server_ip='116.177.248.126' and common_client_ip='120.242.132.200'",
"common_server_ip='116.177.248.126' and common_stream_trace_id=1153021139190754263",
"common_client_ip='120.242.132.200' and common_server_ip='116.177.248.126'",
"http_domain='qq.com' or common_server_ip='120.239.72.226'",
"common_server_port not in (80,443)",
"http_domain not like '%qq.com'"
]
},
{
"name":"@index_filter",
"value": [
"common_log_id=1153021139190754263",
"common_client_ip='118.180.48.74'",
"common_client_ip='120.242.132.200'",
"common_server_ip='114.114.114.114'",
"common_server_ip!='114.114.114.114'",
"common_server_ip='120.239.72.226'",
"http_domain='qq.com'",
"http_domain!='qq.com'",
"http_domain='yunser.com'",
"ssl_sni='mmbiz.qpic.cn'",
"ssl_sni='openai.qq.com'",
"common_server_ip like '120.239%'",
"common_server_ip not like '120.239%'",
"common_server_ip like '%114.114%'",
"common_subscriber_id='%test%'",
"http_domain like 'baidu%'",
"http_domain like '%baidu.com%'",
"common_client_ip in ('120.239.72.226','114.114.114.114')",
"common_client_ip not in ('120.239.72.226','114.114.114.114')",
"common_server_ip='116.177.248.126' and notEmpty(http_domain)",
"common_server_ip='116.177.248.126' and common_client_ip='120.242.132.200'",
"common_server_ip='116.177.248.126' and common_stream_trace_id=1153021139190754263",
"common_client_ip='120.242.132.200' and common_server_ip='116.177.248.126'",
"http_domain='qq.com' or common_server_ip='120.239.72.226'"
]
}
]
}

118
TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-queries-template.sql Normal file
View File

@@ -0,0 +1,118 @@
--Q01.Count(1)
select count(1) from session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)
--Q02.All Fields Query (default)
SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) LIMIT 30
--Q03.All Fields Query order by Time desc
SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30
--Q04.All Fields Query order by Time asc
SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time asc LIMIT 30
--Q05.All Fields Query by Filter
SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30
--Q06.Default Fields Query by Filter
SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30
--Q07.All Fields Query (sub query by time)
SELECT * FROM session_record AS session_record WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30
--Q08.All Fields Query (sub query by log id)
SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30
--Q09.Default Field Query (sub query by time)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30
--Q10.Default Field Query (sub query by log id)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( select common_log_id FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)) ORDER BY common_recv_time DESC LIMIT 30
--Q11.Default Field Query by Server IP (sub query by log id with Index Table)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_common_server_ip AS session_record_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30
--Q12.Default Field Query by Client IP (sub query by log id with Index Table)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_common_client_ip AS session_record_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30
--Q13.Default Field Query by Domain (sub query by log id with Index Table)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_http_domain AS session_record_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30
--Q14.All Fields Query by Client IP (sub query by log id with index Table)
SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_common_client_ip AS session_record_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30
--Q15.All Fields Query by Server IP(sub query by log id with index Table)
SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_common_server_ip AS session_record_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30
--Q16.All Fields Query by Domain(sub query by log id with index Table)
SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_http_domain AS session_record_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30
--Q17.Session Logs Sent to Database Trend(Time Grain 5 minute)
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", count(common_log_id) AS "logs" FROM session_record AS session_record WHERE ( ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ) ) GROUP BY "Receive Time" LIMIT 10000
--Q18.Traffic Bandwidth Trend(Time Grain 30 second)
SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 30 SECOND)))) AS stat_time, sum(common_c2s_byte_num) AS bytes_sent, sum(common_s2c_byte_num) AS bytes_received, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets, sum(common_sessions) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time ORDER BY stat_time ASC LIMIT 10000
--Q19.Log Tend by Type (Time Grain 5 minute)
SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) GROUP BY stat_time, common_schema_type ORDER BY stat_time ASC LIMIT 10000
--Q20.Traffic Metrics Analytic
SELECT round(sum(common_s2c_byte_num) * 8 / 300,2) AS trafficInBits, round(sum(common_c2s_byte_num) * 8 / 300,2) AS trafficOutBits, round(sum(common_s2c_byte_num + common_c2s_byte_num) * 8 / 300,2) AS trafficTotalBits, round(sum(common_s2c_pkt_num) / 300,2) AS trafficInPackets, round(sum(common_c2s_pkt_num) / 300,2) AS trafficOutPackets, round(sum(common_s2c_pkt_num + common_c2s_pkt_num) / 300,2) AS trafficTotalPackets, round(sum(common_sessions) / 300,2) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)
--Q21.Traffic Endpoints Metrics Trend(Time Grain 5 minute)
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", uniq(common_internal_ip) AS "Unique Internal IP", uniq(common_external_ip) AS "Unique External IP", uniq(common_subscriber_id) AS "Unique Subscriber ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000
--Q22.Endpoint Unique Num by L4 Protocol
SELECT 'all' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) UNION ALL SELECT 'tcp' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) UNION ALL SELECT 'UDP' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_UDP', 'IPv6_UDP' )
--Q23.One-sided Connection Trend(Time Grain 5 minute)
SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, (CASE WHEN common_stream_dir = 1 THEN 'c2s' WHEN common_stream_dir = 2 THEN 's2c' WHEN common_stream_dir = 3 THEN 'double' ELSE 'None' END) AS type, sum(common_sessions) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time, common_stream_dir ORDER BY stat_time ASC LIMIT 10000
--Q24. Estimated One-sided Sessions with Bandwidth
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_sessions) AS "sessions", sum(if(common_stream_dir <> 3, common_sessions, 0)) AS "one_side_sessions", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", round(one_side_sessions / sessions, 2) AS one_side_percent FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000
--Q25.Estimated TCP Sequence Gap Loss
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", sum(common_c2s_tcp_lostlen + common_s2c_tcp_lostlen) AS "gap_loss_bytes", round(gap_loss_bytes / bytes, 2) AS gap_loss_percent FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Receive Time" LIMIT 10000
--Q26.Top30 Server IP by Bytes
SELECT "server_ip" AS "server_ip" , SUM(coalesce("bytes",0)) AS "bytes" , SUM(coalesce("bytes_sent",0)) AS "Sent" , SUM(coalesce("bytes_received",0)) AS "Received" , SUM(coalesce("sessions",0)) AS "sessions" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(common_c2s_byte_num+common_s2c_byte_num) AS "bytes" , SUM(coalesce(common_sessions,0)) AS "sessions" , common_server_ip AS "server_ip" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" desc ) GROUP BY "server_ip" ORDER BY "bytes" desc LIMIT 30
--Q27.Top30 Client IP by Sessions
SELECT common_client_ip , COUNT(*) AS sessions FROM session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_client_ip ORDER BY sessions desc LIMIT 0,30
--Q28.Top30 TCP Server Ports by Sessions
SELECT "Server Port" AS "Server Port", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_port AS "Server Port", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Server Port" LIMIT 1048576) GROUP BY "Server Port" ORDER BY "Sessions" DESC LIMIT 30
--Q29.Top30 Domian by Bytes
SELECT "domain" AS "Website Domain" , SUM(coalesce("bytes",0)) AS "Throughput" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "bytes" , http_domain AS "domain" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "domain" ORDER BY "bytes" desc ) GROUP BY "domain" ORDER BY "Throughput" desc LIMIT 30
--Q30.Top30 Endpoint Devices by Bandwidth
SELECT "device_id" AS "device_id", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, common_device_id AS "device_id" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 30
--Q31.Top30 Domain by Unique Client IP
SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Client IP", 0)) AS "Client IP" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Client IP" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 30
--Q32.Top100 Most Time Consuming Domains
SELECT "Domain" AS "Domain", avg(coalesce("Avg Establish Latency(ms)", 0)) AS "Avg Establish Latency(ms)" FROM (SELECT http_domain AS "Domain", avg(coalesce(common_establish_latency_ms, 0)) AS "Avg Establish Latency(ms)" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Avg Establish Latency(ms)" DESC LIMIT 100
--Q33.Top30 Sources by Sessions
SELECT "source" AS "source", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(common_subscriber_id, ''), nullif(common_client_ip, '')) AS "source", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 30
--Q34.Top30 Destinations by Sessions
SELECT "destination" AS "destination", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(http_domain, ''), nullif(common_server_ip, '')) AS "destination", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 30
--Q35.Top30 Destination Regions by Bandwidth
SELECT "server_location" AS "server_location", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT arrayElement(splitByString(',', common_server_location), length(splitByString(',', common_server_location))) AS "server_location", sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "bytes", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 30
--Q36.Top30 URLS by Sessions
SELECT "Http URL" AS "Http URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "Http URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http URL" LIMIT 1048576) GROUP BY "Http URL" ORDER BY "Sessions" DESC LIMIT 30
--Q37.Top30 Destination Transmission APP by Bandwidth
SELECT "server_ip" AS "server_ip", groupUniqArray(coalesce("trans_app", 0)) AS "trans_app", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", groupUniqArray(concat(common_l4_protocol, '/', toString(common_server_port))) AS "trans_app", common_server_ip AS "server_ip" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 30
--Q38.Browsing Users by Website domains and Sessions
SELECT "Subscriber ID" AS "Subscriber ID", "Http.Domain" AS "Http.Domain", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT http_domain AS "Http.Domain", common_subscriber_id AS "Subscriber ID", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) AND notEmpty(common_subscriber_id) ) GROUP BY "Http.Domain", "Subscriber ID" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Subscriber ID", "Http.Domain" ORDER BY "sessions" DESC LIMIT 10000
--Q39.Top Domain and Server IP by Bytes Sent
SELECT "Http.Domain" AS "Http.Domain" , "Server IP" AS "Server IP" , SUM(coalesce("Bytes Sent",0)) AS "Bytes Sent" FROM ( SELECT common_server_ip AS "Server IP" , http_domain AS "Http.Domain" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "Bytes" , SUM(coalesce(common_c2s_byte_num,0)) AS "Bytes Sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "Bytes Received" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "Server IP" , "Http.Domain" ORDER BY "Bytes" desc LIMIT 1048576 ) GROUP BY "Http.Domain" , "Server IP" ORDER BY "Bytes Sent" desc LIMIT 10000
--Q40.Top30 Website Domains by Client IP and Sessions
SELECT "Http.Domain" AS "Http.Domain", "Client IP" AS "Client IP", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT common_client_ip AS "Client IP", http_domain AS "Http.Domain", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Client IP", "Http.Domain" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Http.Domain", "Client IP" ORDER BY "sessions" DESC LIMIT 10000
--Q41.Domain is Accessed by Unique Client IP Trend(bytes Time Grain 5 minute)
SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , http_domain AS Domain, COUNT(DISTINCT(common_client_ip)) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) AND http_domain IN ( SELECT http_domain FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY SUM(common_s2c_byte_num+common_c2s_byte_num) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , http_domain ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000
--Q42. Domain is Accessed by Unique Client IP Trend(sessions,Time Grain 5 minute)
SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),3600)*3600) AS stat_time , http_domain , uniq (common_client_ip) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start)-604800 AND common_recv_time < toDateTime(@end) AND http_domain IN ( SELECT http_domain FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY COUNT(*) desc LIMIT 5 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 3600)*3600), http_domain ORDER BY stat_time desc LIMIT 10000
--Q43.Bandwidth Trend with Device ID(Time Grain 5 minute)
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", common_device_id AS "Device ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time", "Device ID" LIMIT 10000
--Q44.Internal IP by Sled IP and Sessions
SELECT "Internal IP" AS "Internal IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_internal_ip AS "Internal IP", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Sled IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 10000
--Q45.Bandwidth Trend with Internal IP (Time Grain 5 minute)
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_c2s_pkt_num + common_s2c_pkt_num, 0)) AS "Packets", sum(coalesce(common_sessions, 0)) AS "New Sessions", sum(coalesce(common_c2s_byte_num, 0)) AS "Bytes Sent", sum(coalesce(common_s2c_byte_num, 0)) AS "Bytes Received", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent", sum(coalesce(common_s2c_pkt_num, 0)) AS "Packets Received" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) GROUP BY "Receive Time" LIMIT 10000
--Q46.Top30 Domains Detail with Internal IP
SELECT "Domain" AS "Domain", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_domain AS "Domain", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Sessions" DESC LIMIT 30
--Q47.Top30 URLS Detail with Internal IP
SELECT "URL" AS "URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_url) ) GROUP BY "URL" LIMIT 1048576) GROUP BY "URL" ORDER BY "Sessions" DESC LIMIT 30
--Q48.Top Domains with Unique Client IP and Subscriber ID
SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Unique Client IP", 0)) AS "Unique Client IP", sum(coalesce("Unique Subscriber ID", 0)) AS "Unique Subscriber ID" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Unique Client IP", uniq(common_subscriber_id) AS "Unique Subscriber ID" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Unique Client IP" DESC LIMIT 100
--Q49.Top100 Domains by Packets sent
SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Packets Sent", 0)) AS "Packets Sent" FROM (SELECT http_domain AS "Http.Domain", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Packets Sent" DESC LIMIT 100
--Q50.Internal and External asymmetric traffic
SELECT "Internal IP" AS "Internal IP", "External IP" AS "External IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_external_ip AS "External IP", common_internal_ip AS "Internal IP", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes Sent+Bytes Received", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Sled IP", "External IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "External IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 500
--Q51.Client and Server ASN asymmetric traffic
SELECT "Client ASN" AS "Client ASN", "Server ASN" AS "Server ASN", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_asn AS "Server ASN", common_client_asn AS "Client ASN", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Server ASN", "Client ASN" LIMIT 1048576) GROUP BY "Client ASN", "Server ASN" ORDER BY "Sessions" DESC LIMIT 500
--Q52.Top handshake latency by Website and Client IPs
SELECT "SSL.SNI" AS "SSL.SNI", "Client IP" AS "Client IP", avg(coalesce("Establish Latency(ms)", 0)) AS "Establish Latency(ms)" FROM (SELECT common_client_ip AS "Client IP", ssl_sni AS "SSL.SNI", avg(coalesce(common_establish_latency_ms, 0)) AS "Establish Latency(ms)" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Client IP", "SSL.SNI" LIMIT 1048576) GROUP BY "SSL.SNI", "Client IP" ORDER BY "Establish Latency(ms)" DESC LIMIT 500
--Q53.Domain baidu.com Drill down Client IP
select common_client_ip as "Client IP" , avg(common_establish_latency_ms) as "Establishing Time Mean(ms)", count(1) as Responses,any(common_client_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Client IP" order by Responses desc limit 100
--Q54.Domain baidu.com Drill down Server IP
select common_server_ip as "Server IP" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses,any(common_server_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Server IP" order by Responses desc limit 100
--Q55.Domain baidu.com Drill down URI
select http_url as "URI" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "URI" order by Responses desc limit 100
--Q56.L7 Protocol Metrics
select common_l7_protocol as "Protocol" , uniq(common_client_ip) as "Clients" , uniq(common_server_ip) as "Servers", count(1) as Sessions,sum(common_c2s_byte_num+common_s2c_byte_num) as bytes from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and notEmpty(common_l7_protocol) group by common_l7_protocol order by bytes desc
--Q57.L7 Protocol SIP Drill down Client IP
select common_client_ip as "Client IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_client_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Client IP" order by Sessions desc limit 100
--Q58.L7 Protocol SIP Drill down Server IP
select common_server_ip as "Server IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_server_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Server IP" order by Sessions desc limit 100
--Q59.Top5 Server IP keys with Unique Client IPs Trend (Grain 5 minute)
SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , common_server_ip AS server_ip, COUNT(DISTINCT(common_client_ip)) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_server_ip IN ( SELECT common_server_ip FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_server_ip ORDER BY count(*) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , server_ip ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000

11
TSG发布版本更新记录/TSG-21.11/qgw/schema/columns_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "columns_cluster",
"fields": [
{
"name": "database",
"type": "string"
}
]
}

11
TSG发布版本更新记录/TSG-21.11/qgw/schema/disks_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "disks_cluster",
"fields": [
{
"name": "name",
"type": "string"
}
]
}

348
TSG发布版本更新记录/TSG-21.11/qgw/schema/dos_event.json Normal file
View File

@@ -0,0 +1,348 @@
{
"type": "record",
"name": "dos_event",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "log_id",
"partition_key": "start_time",
"functions": {
"aggregation": [
{
"name": "COUNT",
"label": "COUNT",
"function": "count(expr)"
},
{
"name": "COUNT_DISTINCT",
"label": "COUNT_DISTINCT",
"function": "count(distinct expr)"
},
{
"name": "AVG",
"label": "AVG",
"function": "avg(expr)"
},
{
"name": "SUM",
"label": "SUM",
"function": "sum(expr)"
},
{
"name": "MAX",
"label": "MAX",
"function": "max(expr)"
},
{
"name": "MIN",
"label": "MIN",
"function": "min(expr)"
}
],
"operator": [
{
"name": "=",
"label": "=",
"function": "expr = value"
},
{
"name": "!=",
"label": "!=",
"function": "expr != value"
},
{
"name": ">",
"label": ">",
"function": "expr > value"
},
{
"name": "<",
"label": "<",
"function": "expr < value"
},
{
"name": ">=",
"label": ">=",
"function": "expr >= value"
},
{
"name": "<=",
"label": "<=",
"function": "expr <= value"
},
{
"name": "has",
"label": "HAS",
"function": "has(expr, value)"
},
{
"name": "in",
"label": "IN",
"function": "expr in (values)"
},
{
"name": "not in",
"label": "NOT IN",
"function": "expr not in (values)"
},
{
"name": "like",
"label": "LIKE",
"function": "expr like value"
},
{
"name": "not like",
"label": "NOT LIKE",
"function": "expr not like value"
},
{
"name": "notEmpty",
"label": "NOT EMPTY",
"function": "notEmpty(expr)"
},
{
"name": "empty",
"label": "EMPTY",
"function": "empty(expr)"
}
]
},
"schema_query": {
"references": {
"aggregation": [
{
"type": "int",
"functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN"
},
{
"type": "long",
"functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN"
},
{
"type": "float",
"functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN"
},
{
"type": "double",
"functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN"
},
{
"type": "string",
"functions": "COUNT,COUNT_DISTINCT"
},
{
"type": "date",
"functions": "COUNT,COUNT_DISTINCT,MAX,MIN"
},
{
"type": "timestamp",
"functions": "COUNT,COUNT_DISTINCT,MAX,MIN"
}
],
"operator": [
{
"type": "int",
"functions": "=,!=,>,<,>=,<=,in,not in"
},
{
"type": "long",
"functions": "=,!=,>,<,>=,<=,in,not in"
},
{
"type": "float",
"functions": "=,!=,>,<,>=,<="
},
{
"type": "double",
"functions": "=,!=,>,<,>=,<="
},
{
"type": "string",
"functions": "=,!=,in,not in,like,not like,notEmpty,empty"
},
{
"type": "date",
"functions": "=,!=,>,<,>=,<="
},
{
"type": "timestamp",
"functions": "=,!=,>,<,>=,<="
},
{
"type": "array",
"functions": "has"
}
]
}
},
"default_columns": [
"log_id",
"attack_type",
"source_ip_list",
"destination_ip",
"severity",
"start_time",
"end_time",
"packet_rate",
"bit_rate",
"session_rate"
]
},
"fields": [
{
"name": "start_time",
"label": "Start Time",
"doc": {
"allow_query": "true",
"constraints": {
"type": "timestamp"
}
},
"type": "long"
},
{
"name": "end_time",
"label": "End Time",
"doc": {
"constraints": {
"type": "timestamp"
}
},
"type": "long"
},
{
"name": "log_id",
"label": "Log ID",
"doc": {
"allow_query": "true",
"format": {
"functions": "snowflake_id"
}
},
"type": "long"
},
{
"name": "attack_type",
"label": "Attack Type",
"doc": {
"allow_query": "true",
"constraints": {
"operator_functions": "=,!="
},
"data": [
{
"code": "TCP SYN Flood",
"value": "TCP SYN Flood"
},
{
"code": "UDP Flood",
"value": "UDP Flood"
},
{
"code": "ICMP Flood",
"value": "ICMP Flood"
},
{
"code": "DNS Flood",
"value": "DNS Flood"
},
{
"code": "DNS Amplification",
"value": "DNS Amplification"
}
]
},
"type": "string"
},
{
"name": "severity",
"label": "Severity",
"doc": {
"allow_query": "true",
"constraints": {
"operator_functions": "=,!="
},
"data": [
{
"code": "Critical",
"value": "Critical"
},
{
"code": "Severe",
"value": "Severe"
},
{
"code": "Major",
"value": "Major"
},
{
"code": "Warning",
"value": "Warning"
},
{
"code": "Minor",
"value": "Minor"
}
]
},
"type": "string"
},
{
"name": "conditions",
"label": "Conditions",
"type": "string"
},
{
"name": "destination_ip",
"label": "Destination IP",
"doc": {
"allow_query": "true"
},
"type": "string"
},
{
"name": "destination_country",
"label": "Destination Country",
"type": "string"
},
{
"name": "source_ip_list",
"label": "Source IPs",
"type": "string"
},
{
"name": "source_country_list",
"label": "Source Countries",
"type": "string"
},
{
"name": "session_rate",
"label": "Sessions/s",
"doc": {
"constraints": {
"type": "sessions/sec"
}
},
"type": "long"
},
{
"name": "packet_rate",
"label": "Packets/s",
"doc": {
"constraints": {
"type": "packets/sec"
}
},
"type": "long"
},
{
"name": "bit_rate",
"label": "Bits/s",
"doc": {
"constraints": {
"type": "bits/sec"
}
},
"type": "long"
}
]
}

21
TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-filter.json Normal file
View File

@@ -0,0 +1,21 @@
{
"version": "1.0",
"name": "druid-Raw",
"namespace": "druid",
"filters": [
{
"name":"@start",
"value": "'2021-10-19 10:00:00'"
},
{
"name":"@end",
"value": "'2021-10-20 11:00:00'"
},
{
"name":"@common_filter",
"value": [
"common_client_ip='192.168.44.21'and common_server_port=443"
]
}
]
}

92
TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-queries-template.sql Normal file
View File

@@ -0,0 +1,92 @@
--Q01.All Security Event Hits
select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end group by policy_id
--Q02.Security Event Hits with Policy ID 0
select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id in (0) group by policy_id
--Q03.All Security Event Hits Trend by 5min A
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from security_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 10000
--Q04.Security Event Hit Time（first and last time) A
select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from security_event_hits_log where policy_id in (0) group by policy_id
--Q05.Top 200 Security Policies
select policy_id, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id order by hits desc limit 200
--Q06.Top 200 Security Policies with Action
select policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=@start and __time <@end group by policy_id, action order by hits desc limit 200
--Q07.All Proxy Event Hits
select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id
--Q08.Proxy Event Hits with Policy ID 0
select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=0 group by policy_id
--Q09.All Proxy Event Hits Trend by 5min A
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 10000
--Q10.Proxy Event Hit Time（first and last time) A
select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from proxy_event_hits_log where policy_id in (0) group by policy_id
--Q11.Top 200 Proxy Policies
select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id order by hits desc limit 200
--Q12.Top 200 Proxy Policies with sub Action
select policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id, sub_action order by hits desc limit 200
--Q13.Proxy Action Hits
select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by sub_action
--Q14.Proxy Action Hits Trend by 5min
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') , sub_action limit 10000
--Q15.Traffic Metrics Pinning Hits
SELECT sum(not_pinning_num) AS sessions, 'notPinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(pinning_num) AS sessions, 'pinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(maybe_pinning_num) AS sessions, 'maybePinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end
--Q16.Traffic Metrics Pinning Trend by 5Min
SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000
--Q17.Traffic Metrics Not Pinning Trend by 5Min
SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(not_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time>= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000
--Q18.Traffic Metrics Maybe Pinning Trend by 5Min
SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(maybe_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000
--Q19.Traffic Metrics Throughput Bytes IN/OUT
select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q20. Traffic Metrics Throughput Packets IN/OUT
select sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q21.Traffic Metrics New Sessions
select sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q22.Traffic Metrics Bandwidth Bytes IN/OUT
select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')
--Q23.Traffic Metrics Bandwidth Packets IN/OUT
select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')
--Q24.Traffic Metrics New Sessions Trend by 5Min
select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')
--Q25.Traffic Metrics New and Live Sessions
select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q26.Traffic Metrics New and Live Sessions Trend by 5Min
select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')
--Q27.Traffic Metrics Security Throughput Bytes
select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end
--Q28.Traffic Metrics Security Throughput Packets
select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q29.Traffic Metrics Security Throughput Sessions
select sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q30.Traffic Metrics Security Bandwidth Bytes by 5Min
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_bytes' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_bytes' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_bytes' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_bytes' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_bytes' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')
--Q31.Traffic Metrics Security Bandwidth Packets by 5Min
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_packets' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_packets' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_packets' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_packets' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')
--Q32.Traffic Metrics Security Sessions Trend by 5Min
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_conn_num' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_conn_num' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_conn_num' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_conn_num' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_conn_num' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')
--Q33.Top 100 Client IP by Sessions
select source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_client_ip_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by source order by sessions desc limit 100
--Q34.Top 100 Server IP by Sessions
select destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_server_ip_log where __time >= @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100
--Q35.Top 100 Internal IP by Sessions
select source as internal_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by source order by sessions desc limit 100
--Q36.Top 100 External IP by Sessions
select destination as external_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >= @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100
--Q37.Top 100 Domain by Bytes
select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='bytes' group by domain order by bytes desc limit 100
--Q38.Top 100 Subscriber ID by Sessions
select subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by subscriber_id order by sessions desc limit 100
--Q39.Top 100 Hit URLS by hits
select url,sum(session_num) as hits from top_urls_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by url order by hits desc limit 100
--Q40.Proxy Event Unique ISP
SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end group by policy_id
--Q41.Traffic Composition Metrics
SELECT APPROX_COUNT_DISTINCT_DS_HLL(ip_object) AS uniq_client_ip, SUM(one_sided_connections) AS one_sided_connections, SUM(uncategorized_bytes) AS total_uncategorized_bytes, SUM(fragmentation_packets) AS fragmentation_packets, SUM(sequence_gap_loss) AS sequence_gap_loss_bytes, SUM(s2c_byte_num+c2s_byte_num) AS summaryTotalBytes, SUM(s2c_pkt_num+c2s_pkt_num) AS summaryTotalPackets, SUM(sessions) AS summarySessions FROM traffic_summary_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end LIMIT 1
--Q42.Traffic Composition Throughput
(SELECT SUM(c2s_byte_num + s2c_byte_num) as total_bytes, SUM(sessions) as total_sessions, (SUM(c2s_byte_num + s2c_byte_num) * 8)/((TIMESTAMP_TO_MILLIS(TIMESTAMP @end )-TIMESTAMP_TO_MILLIS(TIMESTAMP @start ))/1000) AS data_rate FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' LIMIT 1) UNION ALL ( SELECT SUM(sessions), 0, 0 FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' GROUP BY __time ORDER BY __time DESC LIMIT 1 )
--Q43.Traffic Composition Protocol Tree
SELECT protocol_id, SUM(sessions) as sessions,SUM(c2s_byte_num) as c2s_byte_num, SUM(c2s_pkt_num) as c2s_pkt_num, SUM(s2c_byte_num) as s2c_byte_num, SUM(s2c_pkt_num) as s2c_pkt_num FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end GROUP BY protocol_id
--Q44.System Quota
SELECT log_type, SUM(used_size) as used_size, SUM(max_size) * 7/10 as max_size, TIME_FORMAT(LATEST(last_storage) * 1000,'YYYY-MM-dd') as first_storage FROM ( SELECT log_type, LATEST(used_size) as used_size, LATEST(max_size) as max_size, LATEST(last_storage) as last_storage FROM sys_storage_log WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '1' HOUR AND data_center != '' GROUP BY data_center,log_type ) GROUP BY log_type
--Q45.System Quota Daily Trend
select TIME_FORMAT(__time,'YYYY-MM-dd') as stat_time,log_type as type, sum(aggregate_size) as used_size from sys_storage_log where __time >= @start and __time < @end group by TIME_FORMAT(__time,'YYYY-MM-dd'), log_type
--Q46.Traffic Statistics(Metrics01)
select sum(total_hit_sessions) as total_hit_sessions, sum(total_bytes_transferred) as total_bytes_transferred, sum(total_packets_transferred) as total_packets_transferred, sum(total_new_sessions) as total_new_sessions , sum(total_close_sessions) as total_close_sessions, sum(average_new_sessions_per_second) as average_new_sessions_per_second , sum(average_bytes_per_second) as average_bytes_per_second , sum(average_packets_per_second) as average_packets_per_second , COUNT(DISTINCT(device_id)) as device_num, sum(live_sessions) as average_live_sessions from ( select device_id, sum(intercept_conn_num + monitor_conn_num + deny_conn_num + allow_conn_num) as total_hit_sessions, sum(total_in_bytes + total_out_bytes) as total_bytes_transferred, sum(total_in_packets + total_out_packets) as total_packets_transferred, sum(new_conn_num) as total_new_sessions, sum(close_conn_num) as total_close_sessions, avg(nullif(new_conn_num, 0))/ 5 as average_new_sessions_per_second, avg(nullif(total_in_bytes + total_out_bytes, 0))* 8 / 5 as average_bytes_per_second, avg(nullif(total_in_packets + total_out_packets, 0))/ 5 as average_packets_per_second, avg(nullif(established_conn_num, 0)) as live_sessions from traffic_metrics_log where __time >= @start and __time < @end group by device_id)

53
TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-filter.json Normal file
View File

@@ -0,0 +1,53 @@
{
"version": "1.0",
"name": "Engine-Raw",
"namespace": "Engine",
"filters": [
{
"name":"@start",
"value": "'2021-10-19 10:00:00'"
},
{
"name":"@end",
"value": "'2021-10-20 11:00:00'"
},
{
"name":"@common_filter",
"value": [
"common_log_id=1153021139190754263",
"common_client_ip='36.189.226.21'",
"common_internal_ip='223.116.37.192'",
"common_server_ip='8.8.8.8'",
"common_external_ip='111.10.53.14'",
"common_client_port=52607",
"common_server_port=443",
"common_c2s_pkt_num>5",
"common_s2c_pkt_num>5",
"common_c2s_byte_num>100",
"common_s2c_byte_num<200",
"common_schema_type='DNS'",
"common_establish_latency_ms>200",
"common_con_duration_ms>10000",
"common_stream_trace_id=1153021139190754263",
"common_tcp_client_isn=2857077935",
"common_tcp_server_isn=0",
"http_domain='microsoft.com'",
"mail_account='abc@xx.com'",
"mail_subject='test'",
"dns_qname='qbwup.imtt.qq.com'",
"ssl_sni='note.youdao.com'",
"ssl_con_latency_ms>100",
"ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'",
"common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'",
"common_server_ip='111.10.53.14' and common_server_port=443",
"mail_account like 'abc@%'",
"http_domain like '%baidu.com%'",
"ssl_sni like '%youdao.com'",
"common_client_ip in ('36.189.226.21','111.10.53.14')",
"common_server_port not in (80,443)",
"notEmpty(http_domain)",
"http_domain not like '%microsoft.com'"
]
}
]
}

96
TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-queries-template.sql Normal file
View File

@@ -0,0 +1,96 @@
--Q01.CK DateTime
select toDateTime(common_recv_time) as common_recv_time from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 20
--Q02.Standard DateTime
select FROM_UNIXTIME(common_recv_time) as common_recv_time from session_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) limit 20
--Q03.count(1)
select count(1) from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end)
--Q04.count(*)
select count(*) from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end)
--Q05.UDF APPROX_COUNT_DISTINCT_DS_HLL
SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end and policy_id=0 group by policy_id
--Q06.UDF TIME_FLOOR_WITH_FILL
select TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','previous') as stat_time from session_record where common_recv_time > @start and common_recv_time < @end group by stat_time
--Q07.UDF GEO IP
select IP_TO_GEO(common_client_ip) as geo,IP_TO_CITY(common_server_ip) as city,IP_TO_COUNTRY(common_server_ip) as country from session_record limit 10
--Q08.Special characters
select * from session_record where (common_protocol_label ='/$' or common_client_ip like'%') limit 10
--Q09.Federation Query
select * from (select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','zero')) as stat_time from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by stat_time order by stat_time asc)
--Q10.Catalog Database
select * from tsg_galaxy_v3.session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 20
--Q11.Session Record Logs
select * from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) AND @common_filter order by common_recv_time desc limit 20
--Q12.Live Session Record Logs
select * from interim_session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) AND @common_filter order by common_recv_time desc limit 20
--Q13.Transaction Record Logs
select * from transaction_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) order by common_recv_time desc limit 20
--Q14.Security Event Logs
select * from security_event where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) AND @common_filter order by common_recv_time desc limit 0,20
--Q15.Proxy Event Logs
select * from proxy_event where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20
--Q16.Radius Record Logs
select * from radius_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20
--Q17.GTPC Record Logs
select * from gtpc_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20
--Q18.Security Event Logs with fields
select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_log_id,common_policy_id,common_subscriber_id,common_client_ip,common_client_port,common_l4_protocol,common_address_type,common_server_ip,common_server_port,common_action,common_direction,common_sled_ip,common_client_location,common_client_asn,common_server_location,common_server_asn,common_c2s_pkt_num,common_s2c_pkt_num,common_c2s_byte_num,common_s2c_byte_num,common_schema_type,common_sub_action,common_device_id, FROM_UNIXTIME(common_start_time) as common_start_time, FROM_UNIXTIME(common_end_time) as common_end_time,common_establish_latency_ms,common_con_duration_ms,common_stream_dir,common_stream_trace_id,http_url,http_host,http_domain,http_request_body,http_response_body,http_cookie,http_referer,http_user_agent,http_content_length,http_content_type,http_set_cookie,http_version,http_response_latency_ms,http_action_file_size,http_session_duration_ms,mail_protocol_type,mail_account,mail_from_cmd,mail_to_cmd,mail_from,mail_to,mail_cc,mail_bcc,mail_subject,mail_attachment_name,mail_eml_file,dns_message_id,dns_qr,dns_opcode,dns_aa,dns_tc,dns_rd,dns_ra,dns_rcode,dns_qdcount,dns_ancount,dns_nscount,dns_arcount,dns_qname,dns_qtype,dns_qclass,dns_cname,dns_sub,dns_rr,ssl_sni,ssl_san,ssl_cn,ssl_pinningst,ssl_intercept_state,ssl_server_side_latency,ssl_client_side_latency,ssl_server_side_version,ssl_client_side_version,ssl_cert_verify,ssl_error,quic_version,quic_sni,quic_user_agent,ftp_account,ftp_url,ftp_content from security_event where common_recv_time >= @start and common_recv_time < @end order by common_recv_time desc limit 10000
--Q19.Radius ON/OFF Logs For Frame IP
select framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >=toDateTime(@start) and event_timestamp <toDateTime(@end) group by framed_ip limit 20
--Q20.Radius ON/OFF Logs For Account
select account, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >= @start and event_timestamp < @end group by account
--Q21.Radius ON/OFF Logs total Account number
select count(distinct(framed_ip)) as active_ip_num , sum(acct_session_time) as online_duration from (select any(framed_ip) as framed_ip ,max(acct_session_time) as acct_session_time from radius_onff_log where account='000jS' and event_timestamp >= @start and event_timestamp < @end group by acct_session_id)
--Q22.Radius ON/OFF Logs Account Access Detail
select max(if(acct_status_type=1,event_timestamp,0)) as start_time,max(if(acct_status_type=2,event_timestamp,0)) as end_time, any(framed_ip) as ip,max(acct_session_time) as online_duration from radius_onff_log where event_timestamp >= @start and event_timestamp < @end group by acct_session_id order by start_time desc limit 200
--Q23.Report for Client IP
select common_client_ip, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@end)) group by common_client_ip order by sessions desc limit 0,100
--Q24.Report for Server IP
select common_server_ip, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by common_server_ip order by sessions desc limit 0,100
--Q25.Report for SSL SNI
select ssl_sni, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by ssl_sni order by sessions desc limit 0,100
--Q26.Report for SSL APP
select common_app_label as applicaiton, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by applicaiton order by sessions desc limit 0,100
--Q27.Report for Domains
select http_domain AS domain,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(domain) GROUP BY domain ORDER BY bytes DESC LIMIT 100
--Q28.Report for Domains with unique Client IP
select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_domain, uniq (common_client_ip) as nums from session_record where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_domain in (select http_domain from session_record where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_domain) group by http_domain order by SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_domain order by stat_time asc limit 500
--Q29. Report for HTTP Host
SELECT http_host as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) GROUP BY host ORDER BY bytes DESC limit 100 union all SELECT 'totals' as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes, SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes, SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host)
--Q30.Report for HTTP/HTTPS URLS with Sessions
SELECT http_url AS url,count(*) AS sessions FROM proxy_event WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) GROUP BY url ORDER BY sessions DESC LIMIT 100
--Q31.Report for HTTP/HTTPS URLS with UNIQUE Client IP
select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_url, count(distinct(common_client_ip)) as nums from proxy_event where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_url IN (select http_url from proxy_event where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) group by http_url order by count(*) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_url order by stat_time asc limit 500
--Q32.Report for Subscriber ID with Sessions
select common_subscriber_id as user, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) group by common_subscriber_id order by sessions desc limit 0,100
--Q33.Report for Subscriber ID with Bandwidth
SELECT common_subscriber_id as user,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) GROUP BY user ORDER BY bytes DESC LIMIT 100
--Q34.Report Unique Endpoints
select uniq(common_client_ip) as "Client IP",uniq(common_server_ip) as "Server IP",uniq(common_internal_ip) as "Internal IP",uniq(common_external_ip) as "External IP",uniq(http_domain) as "Domain",uniq(ssl_sni) as "SNI" from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start))
--Q35.TopN Optimizer
SELECT http_url AS url, SUM(common_sessions) AS sessions FROM session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_url) GROUP BY http_url ORDER BY sessions DESC limit 10
--Q36.All Security Event Hits Trend by 5min B
select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, sum(hits) as hits from security_event_hits_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') limit 10000
--Q37.Security Event Hit Time（first and last time) B
select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from security_event_hits_log where policy_id in (0) group by policy_id
--Q38.All Proxy Event Hits Trend by 5min B
select FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end group by FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) limit 10000
--Q39.Proxy Event Hit Time（first and last time) B
select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from proxy_event_hits_log where policy_id in (0) group by policy_id
--Q40.Traffic Composition Protocol Tree Trend
(SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes from traffic_protocol_stat_log where __time >= TIMESTAMP @start AND __time < TIMESTAMP @end and protocol_id = 'ETHERNET' group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id order by stat_time asc) union all (SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes from traffic_protocol_stat_log where __time >= TIMESTAMP @start AND __time < TIMESTAMP @end and protocol_id like CONCAT('ETHERNET','.%') and LENGTH(protocol_id) = LENGTH(REPLACE(protocol_id,'.','')) + 1 + 0 group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id order by stat_time asc)
--Q41.Traffic Metrics Security Action Hits Trend
select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) as statisticTime, sum(default_in_bytes + default_out_bytes) as default_bytes, sum(default_in_packets + default_out_packets) as default_packets, sum(default_conn_num) as default_sessions, sum(allow_in_bytes + allow_out_bytes) as allow_bytes, sum(allow_in_packets + allow_out_packets) as allow_packets, sum(allow_conn_num) as allow_sessions, sum(deny_in_bytes + deny_out_bytes) as deny_bytes, sum(deny_in_packets + deny_out_packets) as deny_packets, sum(deny_conn_num) as deny_sessions, sum(monitor_in_bytes + monitor_out_bytes) as monitor_bytes, sum(monitor_in_packets + monitor_out_packets) as monitor_packets, sum(monitor_conn_num) as monitor_sessions, sum(intercept_in_bytes + intercept_out_bytes) as intercept_bytes, sum(intercept_in_packets + intercept_out_packets) as intercept_packets, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >= @start and __time < @end group by FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) limit 100000
--Q42.Traffic Metrics Proxy Action Hits Trend
SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) AS statisticTime,SUM(intcp_allow_num) AS intercept_allow_conn_num,SUM(intcp_mon_num) AS intercept_monitor_conn_num,SUM(intcp_deny_num) AS intercept_deny_conn_num,SUM(intcp_rdirt_num) AS intercept_redirect_conn_num,SUM(intcp_repl_num) AS intercept_replace_conn_num,SUM(intcp_hijk_num) AS intercept_hijack_conn_num,SUM(intcp_ins_num) AS intercept_insert_conn_num FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'PT1800S', 'zero')) LIMIT 100000
--Q43.Traffic Statistics(Metrics02)
select FROM_UNIXTIME(stat_time) as max_active_date_by_sessions, total_live_sessions as max_live_sessions from ( select stat_time, sum(live_sessions) as total_live_sessions from ( select TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'P1D') as stat_time, device_id, avg(established_conn_num) as live_sessions from traffic_metrics_log where __time >= @start and __time<@end group by TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'P1D'), device_id) group by stat_time order by total_live_sessions desc limit 1 )
--Q44.Traffic Summary(Bandwidth Trend)
select * from ( select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time,'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'), 'traffic_in_bytes' union all select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time,'traffic_out_bytes' as type,sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'),'traffic_out_bytes' ) order by stat_time asc limit 100000
--Q45.Traffic Summary(Sessions Trend)
select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time, 'total_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'), 'total_conn_num' order by stat_time asc limit 10000
--Q46.Domain Baidu.com Metrics
select FROM_UNIXTIME(min(common_recv_time)) as "First Seen" , FROM_UNIXTIME(max(common_recv_time)) as "Last Seen" , median(http_response_latency_ms) as "Server Processing Time Median(ms)", count(1) as Responses,any(common_server_location) as Location from session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND http_domain='baidu.com'
--Q47.TIME_FLOOR_WITH_FILL 01
select "Device Group" as "Device Group" ,"Data Center" as "Data Center" ,FROM_UNIXTIME("End Time") as "End Time" , sum("counter") as "counter" from (select common_device_group as "Device Group" ,common_data_center as "Data Center" ,TIME_FLOOR_WITH_FILL (common_end_time,'PT1H','zero') as "End Time" ,count(common_log_id) as "counter" from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by "Device Group","Data Center","End Time") group by "Device Group" ,"Data Center" ,"End Time" order by "End Time" asc limit 5
--Q48.TIME_FLOOR_WITH_FILL 02
select FROM_UNIXTIME("End Time") as "End Time" , sum("counter") as "counter" from (select common_device_group as "Device Group" ,common_data_center as "Data Center" ,TIME_FLOOR_WITH_FILL (common_end_time,'PT1H','zero') as "End Time" ,count(common_log_id) as "counter" ,count(http_domain) as "HTTP.Domain" from security_event where ((common_recv_time >= toDateTime('2021-10-19 00:00:00') and common_recv_time < toDateTime('2021-10-20 00:00:00')) ) AND ( ( common_action = 2 ) ) group by "Device Group","Data Center","End Time") group by "End Time" order by "End Time" asc

1184
TSG发布版本更新记录/TSG-21.11/qgw/schema/gtpc_record.json Normal file
View File

File diff suppressed because it is too large Load Diff

2313
TSG发布版本更新记录/TSG-21.11/qgw/schema/interim_session_record.json Normal file
View File

File diff suppressed because it is too large Load Diff

169
TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_interim.json Normal file
View File

@@ -0,0 +1,169 @@
{
"type": "record",
"name": "liveChart_interim",
"in": "INTERIM-SESSION-RECORD",
"out": "TRAFFIC-PROTOCOL-STAT",
"task": "Protocol-Distribution",
"doc": {
"timestamp": {
"name": "stat_time",
"type": "long"
},
"dimensions": [
{
"name": "protocol_id",
"fieldName": "common_protocol_label",
"type": "string"
},
{
"name": "entrance_id",
"fieldName": "common_entrance_id",
"type": "string"
},
{
"name": "isp",
"fieldName": "common_isp",
"type": "string"
},
{
"name": "data_center",
"fieldName": "common_data_center",
"type": "string"
},
{
"name": "device_group",
"fieldName": "common_device_group",
"type": "string"
}
],
"metrics": [
{
"function": "sum",
"name": "sessions",
"fieldName": "common_sessions",
"type": "long"
},
{
"function": "sum",
"name": "c2s_byte_num",
"fieldName": "common_c2s_byte_diff",
"type": "long"
},
{
"function": "sum",
"name": "s2c_byte_num",
"fieldName": "common_s2c_byte_diff",
"type": "long"
},
{
"function": "sum",
"name": "c2s_pkt_num",
"fieldName": "common_c2s_pkt_diff",
"type": "long"
},
{
"function": "sum",
"name": "s2c_pkt_num",
"fieldName": "common_s2c_pkt_diff",
"type": "long"
},
{
"function": "sum",
"name": "c2s_ipfrag_num",
"fieldName": "common_c2s_ipfrag_num",
"type": "long"
},
{
"function": "sum",
"name": "s2c_ipfrag_num",
"fieldName": "common_s2c_ipfrag_num",
"type": "long"
},
{
"function": "sum",
"name": "c2s_tcp_lostlen",
"fieldName": "common_c2s_tcp_lostlen",
"type": "long"
},
{
"function": "sum",
"name": "s2c_tcp_lostlen",
"fieldName": "common_s2c_tcp_lostlen",
"type": "long"
},
{
"function": "sum",
"name": "c2s_tcp_unorder_num",
"fieldName": "common_c2s_tcp_unorder_num",
"type": "long"
},
{
"function": "sum",
"name": "s2c_tcp_unorder_num",
"fieldName": "common_s2c_tcp_unorder_num",
"type": "long"
},
{
"function": "disCount",
"name": "unique_sip_num",
"fieldName": "common_server_ip",
"type": "long"
},
{
"function": "disCount",
"name": "unique_cip_num",
"fieldName": "common_client_ip",
"type": "long"
}
],
"filters": [
{
"fieldName": "common_protocol_label",
"type": "notempty"
}
],
"transforms": [
{
"function": "combination",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "common_l7_protocol,."
},
{
"function": "combination",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "common_app_label,."
},
{
"function": "flattenSpec",
"name": "data_center",
"fieldName": "common_device_tag",
"parameters": "$.tags[?(@.tag=='data_center')].value"
},
{
"function": "flattenSpec",
"name": "device_group",
"fieldName": "common_device_tag",
"parameters": "$.tags[?(@.tag=='device_group')].value"
},
{
"function": "hierarchy",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "."
}
],
"action": [
{
"label": "Default",
"metrics": "c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num"
}
],
"granularity": {
"type": "period",
"period": "15S"
}
},
"fields": []
}

169
TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_session.json Normal file
View File

@@ -0,0 +1,169 @@
{
"type": "record",
"name": "liveChart_session",
"in": "SESSION-RECORD",
"out": "TRAFFIC-PROTOCOL-STAT",
"task": "Protocol-Distribution",
"doc": {
"timestamp": {
"name": "stat_time",
"type": "long"
},
"dimensions": [
{
"name": "protocol_id",
"fieldName": "common_protocol_label",
"type": "string"
},
{
"name": "entrance_id",
"fieldName": "common_entrance_id",
"type": "string"
},
{
"name": "isp",
"fieldName": "common_isp",
"type": "string"
},
{
"name": "data_center",
"fieldName": "common_data_center",
"type": "string"
},
{
"name": "device_group",
"fieldName": "common_device_group",
"type": "string"
}
],
"metrics": [
{
"function": "sum",
"name": "sessions",
"fieldName": "common_sessions",
"type": "long"
},
{
"function": "sum",
"name": "c2s_byte_num",
"fieldName": "common_c2s_byte_diff",
"type": "long"
},
{
"function": "sum",
"name": "s2c_byte_num",
"fieldName": "common_s2c_byte_diff",
"type": "long"
},
{
"function": "sum",
"name": "c2s_pkt_num",
"fieldName": "common_c2s_pkt_diff",
"type": "long"
},
{
"function": "sum",
"name": "s2c_pkt_num",
"fieldName": "common_s2c_pkt_diff",
"type": "long"
},
{
"function": "sum",
"name": "c2s_ipfrag_num",
"fieldName": "common_c2s_ipfrag_num",
"type": "long"
},
{
"function": "sum",
"name": "s2c_ipfrag_num",
"fieldName": "common_s2c_ipfrag_num",
"type": "long"
},
{
"function": "sum",
"name": "c2s_tcp_lostlen",
"fieldName": "common_c2s_tcp_lostlen",
"type": "long"
},
{
"function": "sum",
"name": "s2c_tcp_lostlen",
"fieldName": "common_s2c_tcp_lostlen",
"type": "long"
},
{
"function": "sum",
"name": "c2s_tcp_unorder_num",
"fieldName": "common_c2s_tcp_unorder_num",
"type": "long"
},
{
"function": "sum",
"name": "s2c_tcp_unorder_num",
"fieldName": "common_s2c_tcp_unorder_num",
"type": "long"
},
{
"function": "disCount",
"name": "unique_sip_num",
"fieldName": "common_server_ip",
"type": "long"
},
{
"function": "disCount",
"name": "unique_cip_num",
"fieldName": "common_client_ip",
"type": "long"
}
],
"filters": [
{
"fieldName": "common_protocol_label",
"type": "notempty"
}
],
"transforms": [
{
"function": "combination",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "common_l7_protocol,."
},
{
"function": "combination",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "common_app_label,."
},
{
"function": "flattenSpec",
"name": "data_center",
"fieldName": "common_device_tag",
"parameters": "$.tags[?(@.tag=='data_center')].value"
},
{
"function": "flattenSpec",
"name": "device_group",
"fieldName": "common_device_tag",
"parameters": "$.tags[?(@.tag=='device_group')].value"
},
{
"function": "hierarchy",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "."
}
],
"action": [
{
"label": "Default",
"metrics": "sessions,c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num,c2s_ipfrag_num,s2c_ipfrag_num,c2s_tcp_lostlen,s2c_tcp_lostlen,c2s_tcp_unorder_num,s2c_tcp_unorder_num"
}
],
"granularity": {
"type": "period",
"period": "15S"
}
},
"fields": []
}

87
TSG发布版本更新记录/TSG-21.11/qgw/schema/meta_data.json Normal file
View File

@@ -0,0 +1,87 @@
{
"metadata": [
{
"namespace": "tsg_galaxy_v3",
"group": "CLICKHOUSE_GROUP",
"tables": [
"radius_onff_log",
"session_record",
"session_record_common_client_ip",
"session_record_common_server_ip",
"session_record_http_domain",
"interim_session_record",
"transaction_record",
"radius_record",
"voip_record",
"gtpc_record",
"security_event",
"proxy_event",
"dos_event",
"active_defence_event",
"sys_packet_capture_event"
]
},
{
"namespace": "elasticsearch",
"group": "ELASTICSEARCH_GROUP",
"tables": [
]
},
{
"namespace": "system",
"group": "CLICKHOUSE_GROUP",
"tables": [
"query_log_cluster",
"tables_cluster",
"columns_cluster",
"disks_cluster",
"parts_cluster",
"processes",
"query_log"
]
},
{
"namespace": "druid",
"group": "DRUID_GROUP",
"tables": [
"top_internal_host_log",
"top_website_domain_log",
"proxy_event_hits_log",
"sys_storage_log",
"security_event_hits_log",
"traffic_protocol_stat_log",
"top_server_ip_log",
"traffic_summary_log",
"traffic_metrics_log",
"top_user_log",
"top_urls_log",
"top_client_ip_log",
"top_external_host_log",
"traffic_app_stat_log",
"traffic_top_destination_ip_metrics_log"
]
},
{
"namespace": "etl",
"group": "ETL_GROUP",
"tables": [
"liveChart_interim",
"liveChart_session"
]
},
{
"namespace":"tsg",
"group":"HBASE_GROUP",
"tables":[
"report_result"
]
},
{
"namespace": "tsg_galaxy",
"group": "HBASE_GROUP",
"tables": [
"relation_account_framedip"
]
}
]
}

11
TSG发布版本更新记录/TSG-21.11/qgw/schema/parts_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "parts_cluster",
"fields": [
{
"name": "name",
"type": "string"
}
]
}

11
TSG发布版本更新记录/TSG-21.11/qgw/schema/processes.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "processes",
"fields": [
{
"name": "query_id",
"type": "string"
}
]
}

1534
TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event.json Normal file
View File

File diff suppressed because it is too large Load Diff

58
TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event_hits_log.json Normal file
View File

@@ -0,0 +1,58 @@
{
"type": "record",
"name": "proxy_event_hits_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "action",
"type": "long"
},
{
"name": "isp",
"type": "string"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "hits",
"type": "long"
},
{
"name": "policy_id",
"type": "long"
},
{
"name": "sub_action",
"type": "string"
},
{
"name": "country",
"type": "string"
},
{
"name": "location",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "ip_object",
"type": "string"
}
]
}

1797
TSG发布版本更新记录/TSG-21.11/qgw/schema/public_schema_info.json Normal file
View File

File diff suppressed because it is too large Load Diff

11
TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "query_log",
"fields": [
{
"name": "query_id",
"type": "string"
}
]
}

11
TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "query_log_cluster",
"fields": [
{
"name": "type",
"type": "string"
}
]
}

37
TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_onff_log.json Normal file
View File

@@ -0,0 +1,37 @@
{
"type": "record",
"name": "radius_onff_log",
"namespace": "tsg_galaxy_v3",
"fields": [
{
"name": "event_timestamp",
"label": "Event Time",
"type": "long"
},
{
"name": "account",
"label": "Account",
"type": "string"
},
{
"name": "framed_ip",
"label": "Framed IP",
"type": "string"
},
{
"name": "acct_session_id",
"label": "Acct Session ID",
"type": "string"
},
{
"name": "acct_status_type",
"label": "Acct Status Type",
"type": "int"
},
{
"name": "acct_session_time",
"label": "Acct Session Time",
"type": "int"
}
]
}

1376
TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_record.json Normal file
View File

File diff suppressed because it is too large Load Diff

7
TSG发布版本更新记录/TSG-21.11/qgw/schema/relation_account_framedip.json Normal file
View File

@@ -0,0 +1,7 @@
{
"type": "record",
"name": "relation_account_framedip",
"namespace": "tsg_galaxy",
"fields": [
]
}

7
TSG发布版本更新记录/TSG-21.11/qgw/schema/report_result.json Normal file
View File

@@ -0,0 +1,7 @@
{
"type": "record",
"name": "report_result",
"namespace": "tsg",
"fields": [
]
}

2424
TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event.json Normal file
View File

File diff suppressed because it is too large Load Diff

42
TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event_hits_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "security_event_hits_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "action",
"type": "long"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "hits",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "isp",
"type": "string"
},
{
"name": "policy_id",
"type": "long"
}
]
}

2359
TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record.json Normal file
View File

File diff suppressed because it is too large Load Diff

71
TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_client_ip.json Normal file
View File

@@ -0,0 +1,71 @@
{
"type": "record",
"name": "session_record_common_client_ip",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"index_key": "common_client_ip"
},
"fields": [
{
"name": "common_log_id",
"type": "long"
},
{
"name": "common_recv_time",
"type": "long"
},
{
"name": "common_server_ip",
"type": "string"
},
{
"name": "common_client_ip",
"type": "string"
},
{
"name": "common_sled_ip",
"type": "string"
},
{
"name": "common_entrance_id",
"type": "int"
},
{
"name": "common_subscriber_id",
"type": "string"
},
{
"name": "common_stream_trace_id",
"type": "long"
},
{
"name": "common_schema_type",
"type": "string"
},
{
"name": "common_client_port",
"type": "int"
},
{
"name": "common_server_port",
"type": "int"
},
{
"name": "common_app_label",
"type": "string"
},
{
"name": "common_direction",
"type": "int"
},
{
"name": "http_domain",
"type": "string"
},
{
"name": "ssl_sni",
"type": "string"
}
]
}

71
TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_server_ip.json Normal file
View File

@@ -0,0 +1,71 @@
{
"type": "record",
"name": "session_record_common_server_ip",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"index_key": "common_server_ip"
},
"fields": [
{
"name": "common_log_id",
"type": "long"
},
{
"name": "common_recv_time",
"type": "long"
},
{
"name": "common_server_ip",
"type": "string"
},
{
"name": "common_client_ip",
"type": "string"
},
{
"name": "common_sled_ip",
"type": "string"
},
{
"name": "common_entrance_id",
"type": "int"
},
{
"name": "common_subscriber_id",
"type": "string"
},
{
"name": "common_stream_trace_id",
"type": "long"
},
{
"name": "common_schema_type",
"type": "string"
},
{
"name": "common_client_port",
"type": "int"
},
{
"name": "common_server_port",
"type": "int"
},
{
"name": "common_app_label",
"type": "string"
},
{
"name": "common_direction",
"type": "int"
},
{
"name": "http_domain",
"type": "string"
},
{
"name": "ssl_sni",
"type": "string"
}
]
}

71
TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_http_domain.json Normal file
View File

@@ -0,0 +1,71 @@
{
"type": "record",
"name": "session_record_http_domain",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"index_key": "http_domain"
},
"fields": [
{
"name": "common_log_id",
"type": "long"
},
{
"name": "common_recv_time",
"type": "long"
},
{
"name": "common_server_ip",
"type": "string"
},
{
"name": "common_client_ip",
"type": "string"
},
{
"name": "common_sled_ip",
"type": "string"
},
{
"name": "common_entrance_id",
"type": "int"
},
{
"name": "common_subscriber_id",
"type": "string"
},
{
"name": "common_stream_trace_id",
"type": "long"
},
{
"name": "common_schema_type",
"type": "string"
},
{
"name": "common_client_port",
"type": "int"
},
{
"name": "common_server_port",
"type": "int"
},
{
"name": "common_app_label",
"type": "string"
},
{
"name": "common_direction",
"type": "int"
},
{
"name": "http_domain",
"type": "string"
},
{
"name": "ssl_sni",
"type": "string"
}
]
}

801
TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_packet_capture_event.json Normal file
View File

@@ -0,0 +1,801 @@
{
"type": "record",
"name": "sys_packet_capture_event",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"partition_key": "common_recv_time"
},
"fields": [
{
"name": "common_recv_time",
"type": "long",
"doc": {
"allow_query": "true",
"constraints": {
"type": "timestamp"
},
"format": {
"functions": "current_timestamp"
}
},
"label": "Receive Time"
},
{
"name": "common_log_id",
"type": "long",
"doc": {
"allow_query": "true",
"format": {
"functions": "snowflake_id"
}
},
"label": "Log ID"
},
{
"name": "common_policy_id",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Policy ID"
},
{
"name": "common_subscriber_id",
"type": "string",
"doc": {
"allow_query": "true"
},
"label": "Subscriber ID"
},
{
"name": "common_imei",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "IMEI"
},
{
"name": "common_imsi",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "IMSI"
},
{
"name": "common_phone_number",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "Phone Number"
},
{
"name": "common_client_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "Client IP"
},
{
"name": "common_internal_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "Internal IP"
},
{
"name": "common_client_port",
"type": "int",
"label": "Client Port"
},
{
"name": "common_l4_protocol",
"type": "string",
"label": "L4 Protocol"
},
{
"name": "common_address_type",
"type": "int",
"doc": {
"data": [
{
"code": "4",
"value": "ipv4"
},
{
"code": "6",
"value": "ipv6"
}
]
},
"label": "Address Type"
},
{
"name": "common_server_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "Server IP"
},
{
"name": "common_server_port",
"type": "int",
"doc": {
"allow_query": "true"
},
"label": "Server Port"
},
{
"name": "common_external_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "External IP"
},
{
"name": "common_action",
"type": "int",
"doc": {
"allow_query": "true",
"data": [
{
"code": "0",
"value": "None"
},
{
"code": "1",
"value": "Monitor"
},
{
"code": "2",
"value": "Intercept"
},
{
"code": "16",
"value": "Deny"
},
{
"code": "128",
"value": "Allow"
}
]
},
"label": "Action"
},
{
"name": "common_direction",
"type": "int",
"doc": {
"data": [
{
"code": "69",
"value": "outbound"
},
{
"code": "73",
"value": "inbound"
}
]
},
"label": "Direction"
},
{
"name": "common_entrance_id",
"type": "int",
"doc": {
"visibility": "disabled"
},
"label": "Entrance ID"
},
{
"name": "common_sled_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "Sled IP"
},
{
"name": "common_client_location",
"type": "string",
"label": "Client Location"
},
{
"name": "common_client_asn",
"type": "string",
"label": "Client ASN"
},
{
"name": "common_server_location",
"type": "string",
"label": "Server Location"
},
{
"name": "common_server_asn",
"type": "string",
"label": "Server ASN"
},
{
"name": "common_sessions",
"type": "long",
"label": "Sessions"
},
{
"name": "common_c2s_pkt_num",
"type": "long",
"label": "Packets Sent"
},
{
"name": "common_s2c_pkt_num",
"type": "long",
"label": "Packets Received"
},
{
"name": "common_c2s_byte_num",
"type": "long",
"label": "Bytes Sent"
},
{
"name": "common_s2c_byte_num",
"type": "long",
"label": "Bytes Received"
},
{
"name": "common_c2s_pkt_diff",
"label": "Packets Sent(Diff)",
"type": "long"
},
{
"name": "common_s2c_pkt_diff",
"label": "Packets Received(Diff)",
"type": "long"
},
{
"name": "common_c2s_byte_diff",
"label": "Bytes Sent(Diff)",
"type": "long"
},
{
"name": "common_s2c_byte_diff",
"label": "Bytes Received(Diff)",
"type": "long"
},
{
"name": "common_service",
"type": "int",
"doc": {
"visibility": "disabled"
},
"label": "Service"
},
{
"name": "common_schema_type",
"type": "string",
"doc": {
"data": [
{
"code": "BASE",
"value": "BASE"
},
{
"code": "HTTP",
"value": "HTTP"
},
{
"code": "MAIL",
"value": "MAIL"
},
{
"code": "DNS",
"value": "DNS"
},
{
"code": "SSL",
"value": "SSL"
},
{
"code": "FTP",
"value": "FTP"
}
],
"visibility": "hidden"
},
"label": "Schema Type"
},
{
"name": "common_user_tags",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "User Tags"
},
{
"name": "common_sub_action",
"type": "string",
"doc": {
"data": [
{
"code": "allow",
"value": "Allow"
},
{
"code": "deny",
"value": "Deny"
},
{
"code": "monitor",
"value": "Monitor"
},
{
"code": "replace",
"value": "Replace"
},
{
"code": "redirect",
"value": "Redirect"
},
{
"code": "insert",
"value": "Insert"
},
{
"code": "hijack",
"value": "Hijack"
}
],
"visibility": "hidden"
},
"label": "Sub Action"
},
{
"name": "common_user_region",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "User Region"
},
{
"name": "common_device_id",
"type": "string",
"label": "Device ID"
},
{
"name": "common_egress_link_id",
"label": "Egress Link ID",
"doc": {
"visibility": "hidden"
},
"type": "int"
},
{
"name": "common_ingress_link_id",
"label": "Ingress Link ID",
"doc": {
"visibility": "hidden"
},
"type": "int"
},
{
"name": "common_isp",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "ISP"
},
{
"name": "common_device_tag",
"type": "string",
"doc": {
"visibility": "hidden",
"format": {
"functions": "flattenSpec,flattenSpec",
"appendTo": "common_data_center,common_device_group",
"param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value"
}
},
"label": "Device Tag"
},
{
"name": "common_data_center",
"label": "Data Center",
"doc": {
"constraints": {
"operator_functions": "=,!="
},
"data": {
"$ref": "device_tag.json#",
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
}
},
"type": "string"
},
{
"name": "common_device_group",
"label": "Device Group",
"doc": {
"constraints": {
"operator_functions": "=,!="
},
"data": {
"$ref": "device_tag.json#",
"key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']",
"value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']"
}
},
"type": "string"
},
{
"name": "common_encapsulation",
"type": "int",
"doc": {
"data": [
{
"code": "0",
"value": "Ethernet"
},
{
"code": "8",
"value": "PPP"
},
{
"code": "12",
"value": "CiscoHDLC"
}
]
},
"label": "Encapsulation"
},
{
"name": "common_app_label",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "Application Label"
},
{
"name": "common_tunnels",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Tunnels"
},
{
"name": "common_protocol_label",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Protocol Label"
},
{
"name": "common_app_id",
"type": "string",
"label": "Application ID",
"doc": {
"visibility": "hidden"
}
},
{
"name": "common_userdefine_app_name",
"label": "User Define APP Name",
"type": "string",
"doc": {
"visibility": "hidden"
}
},
{
"name": "common_app_surrogate_id",
"type": "string",
"label": "Surrogate ID",
"doc": {
"visibility": "hidden"
}
},
{
"name": "common_l7_protocol",
"type": "string",
"label": "L7 Protocol"
},
{
"name": "common_service_category",
"label": "FQDN Category",
"type": {
"type": "array",
"items": "int"
}
},
{
"name": "common_start_time",
"type": "long",
"doc": {
"constraints": {
"type": "timestamp"
},
"visibility": "hidden"
},
"label": "Start Time"
},
{
"name": "common_end_time",
"type": "long",
"doc": {
"constraints": {
"type": "timestamp"
},
"visibility": "hidden"
},
"label": "End Time"
},
{
"name": "common_establish_latency_ms",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Establish Latency(ms)"
},
{
"name": "common_con_duration_ms",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Duration(ms)"
},
{
"name": "common_stream_dir",
"type": "int",
"doc": {
"data": [
{
"code": "1",
"value": "c2s"
},
{
"code": "2",
"value": "s2c"
},
{
"code": "3",
"value": "double"
}
]
},
"label": "Stream Direction"
},
{
"name": "common_address_list",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "Address List"
},
{
"name": "common_has_dup_traffic",
"type": "int",
"doc": {
"data": [
{
"code": "0",
"value": "No"
},
{
"code": "1",
"value": "Yes"
}
],
"visibility": "hidden"
},
"label": "Duplication Traffic"
},
{
"name": "common_stream_error",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Stream Error"
},
{
"name": "common_stream_trace_id",
"type": "long",
"doc": {
"allow_query": "true"
},
"label": "Session ID"
},
{
"name": "common_link_info_c2s",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Link Info(c2s)"
},
{
"name": "common_link_info_s2c",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Link Info(s2c)"
},
{
"name": "common_c2s_ipfrag_num",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Fragmentation Packets(c2s)"
},
{
"name": "common_s2c_ipfrag_num",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Fragmentation Packets(s2c)"
},
{
"name": "common_c2s_tcp_lostlen",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Sequence Gap Loss(c2s)"
},
{
"name": "common_s2c_tcp_lostlen",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Sequence Gap Loss(s2c)"
},
{
"name": "common_c2s_tcp_unorder_num",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Unorder Packets(c2s)"
},
{
"name": "common_s2c_tcp_unorder_num",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Unorder Packets(s2c)"
},
{
"name": "common_c2s_pkt_retrans",
"type": "long",
"label": "Packet Retransmission(c2s)"
},
{
"name": "common_s2c_pkt_retrans",
"type": "long",
"label": "Packet Retransmission(s2c)"
},
{
"name": "common_c2s_byte_retrans",
"type": "long",
"label": "Byte Retransmission(c2s)"
},
{
"name": "common_s2c_byte_retrans",
"type": "long",
"label": "Byte Retransmission(s2c)"
},
{
"name": "common_tcp_client_isn",
"label": "TCP Client ISN",
"doc": {
"visibility": "disabled"
},
"type": "long"
},
{
"name": "common_tcp_server_isn",
"label": "TCP Server ISN",
"doc": {
"visibility": "disabled"
},
"type": "long"
},
{
"name": "common_first_ttl",
"type": "int",
"doc": {
"visibility": "hidden"
},
"label": "First TTL"
},
{
"name": "common_processing_time",
"type": "long",
"doc": {
"constraints": {
"type": "timestamp"
}
},
"label": "Processing Time"
},
{
"name": "common_mirrored_pkts",
"label": "Mirrored Packets",
"type": "long",
"doc": {
"visibility": "hidden"
}
},
{
"name": "common_mirrored_bytes",
"label": "Mirrored Bytes",
"type": "long",
"doc": {
"visibility": "hidden"
}
},
{
"name": "nic_name",
"type": "string",
"label": "Nic Name"
},
{
"name": "origin_source_mac",
"type": "string",
"label": "Origin Source Mac"
},
{
"name": "origin_dest_mac",
"type": "string",
"label": "Origin Dest Mac"
},
{
"name": "packet_url",
"type": "string",
"label": "Packet URL"
},
{
"name": "pcap_storage_task_id",
"type": "int",
"label": "Task ID"
},
{
"name": "pcap_storage_duration",
"type": "int",
"label": "Duration"
}
]
}

38
TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_storage_log.json Normal file
View File

@@ -0,0 +1,38 @@
{
"type": "record",
"name": "sys_storage_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "log_type",
"type": "string"
},
{
"name": "data_center",
"type": "string"
},
{
"name": "max_size",
"type": "long"
},
{
"name": "used_size",
"type": "long"
},
{
"name": "aggregate_size",
"type": "long"
},
{
"name": "last_storage",
"type": "long"
}
]
}

11
TSG发布版本更新记录/TSG-21.11/qgw/schema/tables_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "tables_cluster",
"fields": [
{
"name": "database",
"type": "string"
}
]
}

42
TSG发布版本更新记录/TSG-21.11/qgw/schema/top_client_ip_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_client_ip_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "source",
"type": "string"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
}
]
}

42
TSG发布版本更新记录/TSG-21.11/qgw/schema/top_external_host_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_external_host_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "destination",
"type": "string"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
}
]
}

42
TSG发布版本更新记录/TSG-21.11/qgw/schema/top_internal_host_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_internal_host_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "source",
"type": "string"
}
]
}

42
TSG发布版本更新记录/TSG-21.11/qgw/schema/top_server_ip_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_server_ip_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "destination",
"type": "string"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
}
]
}

22
TSG发布版本更新记录/TSG-21.11/qgw/schema/top_urls_log.json Normal file
View File

@@ -0,0 +1,22 @@
{
"type": "record",
"name": "top_urls_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "url",
"type": "string"
}
]
}

42
TSG发布版本更新记录/TSG-21.11/qgw/schema/top_user_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_user_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "subscriber_id",
"type": "string"
}
]
}

42
TSG发布版本更新记录/TSG-21.11/qgw/schema/top_website_domain_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_website_domain_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "domain",
"type": "string"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
}
]
}

42
TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_app_stat_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "traffic_app_stat_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "app_name",
"type": "string"
},
{
"name": "sub_app_name",
"type": "string"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
}
]
}

214
TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_metrics_log.json Normal file
View File

@@ -0,0 +1,214 @@
{
"type": "record",
"name": "traffic_metrics_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "allow_conn_num",
"type": "long"
},
{
"name": "allow_in_bytes",
"type": "long"
},
{
"name": "allow_in_packets",
"type": "long"
},
{
"name": "allow_out_bytes",
"type": "long"
},
{
"name": "allow_out_packets",
"type": "long"
},
{
"name": "close_conn_num",
"type": "long"
},
{
"name": "default_conn_num",
"type": "long"
},
{
"name": "default_in_bytes",
"type": "long"
},
{
"name": "default_in_packets",
"type": "long"
},
{
"name": "default_out_bytes",
"type": "long"
},
{
"name": "default_out_packets",
"type": "long"
},
{
"name": "deny_conn_num",
"type": "long"
},
{
"name": "deny_in_bytes",
"type": "long"
},
{
"name": "deny_in_packets",
"type": "long"
},
{
"name": "deny_out_bytes",
"type": "long"
},
{
"name": "deny_out_packets",
"type": "long"
},
{
"name": "device_id",
"type": "string"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "intercept_conn_num",
"type": "long"
},
{
"name": "intercept_in_bytes",
"type": "long"
},
{
"name": "intercept_in_packets",
"type": "long"
},
{
"name": "intercept_out_bytes",
"type": "long"
},
{
"name": "intercept_out_packets",
"type": "long"
},
{
"name": "established_conn_num",
"type": "long"
},
{
"name": "monitor_conn_num",
"type": "long"
},
{
"name": "monitor_in_bytes",
"type": "long"
},
{
"name": "monitor_in_packets",
"type": "long"
},
{
"name": "monitor_out_bytes",
"type": "long"
},
{
"name": "monitor_out_packets",
"type": "long"
},
{
"name": "new_conn_num",
"type": "long"
},
{
"name": "total_in_bytes",
"type": "long"
},
{
"name": "total_in_packets",
"type": "long"
},
{
"name": "total_out_bytes",
"type": "long"
},
{
"name": "total_out_packets",
"type": "long"
},
{
"name": "alert_bytes",
"type": "long"
},
{
"name": "hijk_bytes",
"type": "long"
},
{
"name": "ins_bytes",
"type": "long"
},
{
"name": "intcp_allow_num",
"type": "long"
},
{
"name": "intcp_deny_num",
"type": "long"
},
{
"name": "intcp_hijk_num",
"type": "long"
},
{
"name": "intcp_ins_num",
"type": "long"
},
{
"name": "intcp_mon_num",
"type": "long"
},
{
"name": "intcp_rdirt_num",
"type": "long"
},
{
"name": "intcp_repl_num",
"type": "long"
},
{
"name": "maybe_pinning_num",
"type": "long"
},
{
"name": "not_pinning_num",
"type": "long"
},
{
"name": "pinning_num",
"type": "long"
},
{
"name": "ad_cc_bytes",
"type": "long"
},
{
"name": "ad_flood_bytes",
"type": "long"
},
{
"name": "ad_reflection_bytes",
"type": "long"
}
]
}

78
TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_protocol_stat_log.json Normal file
View File

@@ -0,0 +1,78 @@
{
"type": "record",
"name": "traffic_protocol_stat_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "protocol_id",
"type": "string"
},
{
"name": "isp",
"type": "string"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "data_center",
"type": "string"
},
{
"name": "device_group",
"type": "string"
},
{
"name": "sessions",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "c2s_ipfrag_num",
"type": "long"
},
{
"name": "s2c_ipfrag_num",
"type": "long"
},
{
"name": "c2s_tcp_lostlen",
"type": "long"
},
{
"name": "s2c_tcp_lostlen",
"type": "long"
},
{
"name": "c2s_tcp_unorder_num",
"type": "long"
},
{
"name": "s2c_tcp_unorder_num",
"type": "long"
}
]
}

78
TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_summary_log.json Normal file
View File

@@ -0,0 +1,78 @@
{
"type": "record",
"name": "traffic_summary_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "isp",
"type": "string"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "device_group",
"type": "string"
},
{
"name": "data_center",
"type": "string"
},
{
"name": "schema_type",
"type": "string"
},
{
"name": "ip_object",
"type": "string"
},
{
"name": "sessions",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "one_sided_connections",
"type": "long"
},
{
"name": "uncategorized_bytes",
"type": "long"
},
{
"name": "fragmentation_packets",
"type": "long"
},
{
"name": "sequence_gap_loss",
"type": "long"
},
{
"name": "unorder_packets",
"type": "long"
}
]
}

46
TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_top_destination_ip_metrics_log.json Normal file
View File

@@ -0,0 +1,46 @@
{
"type": "record",
"name": "traffic_top_destination_ip_metrics_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "common_sled_ip",
"type": "string"
},
{
"name": "common_data_center",
"type": "string"
},
{
"name": "destination_ip",
"type": "string"
},
{
"name": "attack_type",
"type": "string"
},
{
"name": "session_rate",
"type": "long"
},
{
"name": "packet_rate",
"type": "long"
},
{
"name": "bit_rate",
"type": "long"
},
{
"name": "partition_num",
"type": "long"
}
]
}

1508
TSG发布版本更新记录/TSG-21.11/qgw/schema/transaction_record.json Normal file
View File

File diff suppressed because it is too large Load Diff

95
TSG发布版本更新记录/TSG-21.11/qgw/schema/version.json Normal file
View File

@@ -0,0 +1,95 @@
{
"product": "185 Environment",
"version": "21.11",
"registered": "Geedge",
"updated": "2021-08-01 00:00:00",
"components": {
"oss": [
{
"name": "zookeeper",
"version": "3.4.10",
"licenseType": "Apache License 2.0",
"description": "分布式应用程序协调服务"
},
{
"name": "kafka",
"version": "2.11_1.0.0",
"licenseType": "Apache License 2.0",
"description": "消息队列"
},
{
"name": "habse",
"version": "2.2.3",
"licenseType": "Apache License 2.0",
"description": "用于文件系统和存储Radius数据"
},
{
"name": "flume",
"version": "1.9.0",
"licenseType": "Apache License 2.0",
"description": "日志补全传输"
},
{
"name": "clickhouse",
"version": "20.3.12.112-cluster",
"licenseType": "Apache License 2.0",
"description": "原始日志数据库"
},
{
"name": "druid",
"version": "0.18.1",
"licenseType": "Apache License 2.0",
"description": "分析实时数据并提供低延迟查询的OLAP应用程序"
},
{
"name": "gohangout",
"version": "1.15.2.20210408",
"description": "动态获取原始日志表schema入库程序"
}
],
"apps": [
{
"name": "galaxy-qgw-service",
"version": "345",
"description": "数据平台对外统一查询网关"
},
{
"name": "galaxy-report-service",
"version": "21.04.07",
"description": "自定义报表查询服务"
},
{
"name": "galaxy-hos-service",
"version": "21.07.01",
"description": "对象存储服务"
},
{
"name": "xxl-job-admin",
"version": "v1.3.20210408",
"description": "分布式任务调度平台"
},
{
"name": "xxl-job",
"version": "v1.3.210413-rc1",
"description": "分布式任务调度平台-执行器"
}
],
"tasks": [
{
"name": "flume",
"version": "flume-config-20.08",
"description": "原始日志补全、subscriber更新、Radius上下线功能"
},
{
"name": "druid",
"version": "druid-config-20.08",
"description": "所有分析日志任务"
},
{
"name": "gohangout",
"version": "gohangout-config-20.08",
"description": "原始日志入库、上下线日志入库"
}
]
}
}

1377
TSG发布版本更新记录/TSG-21.11/qgw/schema/voip_record.json Normal file
View File

File diff suppressed because it is too large Load Diff

29
TSG发布版本更新记录/TSG-21.11/record-21.11.txt Normal file
View File

@@ -0,0 +1,29 @@
21.11更新记录
hbase
1.Phoenix集成，hbase新增表tsg_galaxy:relation_account_framedip，tsg_galaxy:relation_framedip_account
2.Phoenix创建视图tsg"."report_result，tsg_galaxy"."relation_account_framedip
ck
1.新增字段
exporter
1. zk ,ck ,kafka,node-exporter
预处理
1.更新jar包及配置
2.新增了subid任务
网关
更新镜像，schema
调度任务
1.更新了任务，sql执行器url添加/，更新了压缩任务
flink
1. 更新了checkpoint配置
nginx
1. 更新了hos配置