From 091c7346c6c6a860ef4aa0813864a3916fee1c7d Mon Sep 17 00:00:00 2001 From: zhanghongqing Date: Mon, 22 Nov 2021 17:01:55 +0800 Subject: [PATCH] 21.11 update --- .../TSG-21.09/dos-detection/.gitkeep | 0 TSG发布版本更新记录/TSG-21.11/.gitkeep | 0 .../TSG-21.11/flink/flink-conf.yml | 39 + .../TSG-21.11/hbase/create_table | 8 +- .../TSG-21.11/hbase/hbase-site.xml | 12 + TSG发布版本更新记录/TSG-21.11/job/xxl_job.sql | 3 + .../TSG-21.11/nginx/nginx.conf | 40 +- .../TSG-21.11/qgw/galaxy-qgw-service.yml | 18 + .../TSG-21.11/qgw/schema/active_defence_event.json | 322 +++ .../TSG-21.11/qgw/schema/ck-filter.json | 99 + .../TSG-21.11/qgw/schema/ck-queries-template.sql | 118 + .../TSG-21.11/qgw/schema/columns_cluster.json | 11 + .../TSG-21.11/qgw/schema/disks_cluster.json | 11 + .../TSG-21.11/qgw/schema/dos_event.json | 348 +++ .../TSG-21.11/qgw/schema/druid-filter.json | 21 + .../TSG-21.11/qgw/schema/druid-queries-template.sql | 92 + .../TSG-21.11/qgw/schema/engine-filter.json | 53 + .../TSG-21.11/qgw/schema/engine-queries-template.sql | 96 + .../TSG-21.11/qgw/schema/gtpc_record.json | 1184 ++++++++ .../TSG-21.11/qgw/schema/interim_session_record.json | 2313 ++++++++++++++++ .../TSG-21.11/qgw/schema/liveChart_interim.json | 169 ++ .../TSG-21.11/qgw/schema/liveChart_session.json | 169 ++ .../TSG-21.11/qgw/schema/meta_data.json | 87 + .../TSG-21.11/qgw/schema/parts_cluster.json | 11 + .../TSG-21.11/qgw/schema/processes.json | 11 + .../TSG-21.11/qgw/schema/proxy_event.json | 1534 +++++++++++ .../TSG-21.11/qgw/schema/proxy_event_hits_log.json | 58 + .../TSG-21.11/qgw/schema/public_schema_info.json | 1797 ++++++++++++ .../TSG-21.11/qgw/schema/query_log.json | 11 + .../TSG-21.11/qgw/schema/query_log_cluster.json | 11 + .../TSG-21.11/qgw/schema/radius_onff_log.json | 37 + .../TSG-21.11/qgw/schema/radius_record.json | 1376 ++++++++++ .../qgw/schema/relation_account_framedip.json | 7 + .../TSG-21.11/qgw/schema/report_result.json | 7 + .../TSG-21.11/qgw/schema/security_event.json | 2424 +++++++++++++++++ .../TSG-21.11/qgw/schema/security_event_hits_log.json | 42 + .../TSG-21.11/qgw/schema/session_record.json | 2359 ++++++++++++++++ .../qgw/schema/session_record_common_client_ip.json | 71 + .../qgw/schema/session_record_common_server_ip.json | 71 + .../qgw/schema/session_record_http_domain.json | 71 + .../qgw/schema/sys_packet_capture_event.json | 801 ++++++ .../TSG-21.11/qgw/schema/sys_storage_log.json | 38 + .../TSG-21.11/qgw/schema/tables_cluster.json | 11 + .../TSG-21.11/qgw/schema/top_client_ip_log.json | 42 + .../TSG-21.11/qgw/schema/top_external_host_log.json | 42 + .../TSG-21.11/qgw/schema/top_internal_host_log.json | 42 + .../TSG-21.11/qgw/schema/top_server_ip_log.json | 42 + .../TSG-21.11/qgw/schema/top_urls_log.json | 22 + .../TSG-21.11/qgw/schema/top_user_log.json | 42 + .../TSG-21.11/qgw/schema/top_website_domain_log.json | 42 + .../TSG-21.11/qgw/schema/traffic_app_stat_log.json | 42 + .../TSG-21.11/qgw/schema/traffic_metrics_log.json | 214 ++ .../qgw/schema/traffic_protocol_stat_log.json | 78 + .../TSG-21.11/qgw/schema/traffic_summary_log.json | 78 + .../traffic_top_destination_ip_metrics_log.json | 46 + .../TSG-21.11/qgw/schema/transaction_record.json | 1508 ++++++++++ .../TSG-21.11/qgw/schema/version.json | 95 + .../TSG-21.11/qgw/schema/voip_record.json | 1377 ++++++++++ .../TSG-21.11/record-21.11.txt | 29 + 59 files changed, 19687 insertions(+), 15 deletions(-) delete mode 100644 TSG发布版本更新记录/TSG-21.09/dos-detection/.gitkeep delete mode 100644 TSG发布版本更新记录/TSG-21.11/.gitkeep create mode 100644 TSG发布版本更新记录/TSG-21.11/flink/flink-conf.yml create mode 100644 TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml create mode 100644 TSG发布版本更新记录/TSG-21.11/job/xxl_job.sql create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/active_defence_event.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-filter.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-queries-template.sql create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/columns_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/disks_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/dos_event.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-filter.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-queries-template.sql create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-filter.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-queries-template.sql create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/gtpc_record.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/interim_session_record.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_interim.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_session.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/meta_data.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/parts_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/processes.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event_hits_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/public_schema_info.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_onff_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_record.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/relation_account_framedip.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/report_result.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event_hits_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_client_ip.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_server_ip.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_http_domain.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_packet_capture_event.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_storage_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/tables_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/top_client_ip_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/top_external_host_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/top_internal_host_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/top_server_ip_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/top_urls_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/top_user_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/top_website_domain_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_app_stat_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_metrics_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_protocol_stat_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_summary_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_top_destination_ip_metrics_log.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/transaction_record.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/version.json create mode 100644 TSG发布版本更新记录/TSG-21.11/qgw/schema/voip_record.json create mode 100644 TSG发布版本更新记录/TSG-21.11/record-21.11.txt diff --git a/TSG发布版本更新记录/TSG-21.09/dos-detection/.gitkeep b/TSG发布版本更新记录/TSG-21.09/dos-detection/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/TSG发布版本更新记录/TSG-21.11/.gitkeep b/TSG发布版本更新记录/TSG-21.11/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/TSG发布版本更新记录/TSG-21.11/flink/flink-conf.yml b/TSG发布版本更新记录/TSG-21.11/flink/flink-conf.yml new file mode 100644 index 0000000..af3f6ca --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/flink/flink-conf.yml @@ -0,0 +1,39 @@ +# flink集群版 配置文件增加checkpoint history + +#============================================================================== +#checkpoint +#============================================================================== +#checkpoint路径 +state.checkpoints.dir: hdfs:///flink/checkpoints +#检查点间隔 +#execution.checkpointing.interval: 300000 +execution.checkpointing.interval: 300000 +#检查点的模式 +execution.checkpointing.mode: EXACTLY_ONCE + +#检查点的超时设置 +execution.checkpointing.timeout: 10min + +#检查点的个数 +state.checkpoints.num-retained: 20 + +state.backend: filesystem + +#============================================================================== +#historyserver +#============================================================================== +#完成的作业到期并从作业存储区中清除的秒数 +jobstore.expiration-time: 7200 + +#flink历史任务 +# 上传已完成作业信息的目录 +jobmanager.archive.fs.dir: hdfs:///history-jobs + +# 监视以下目录中已完成的作业 +historyserver.archive.fs.dir: hdfs:///history-jobs +# +# # 每 10 秒刷新一次 +historyserver.archive.fs.refresh-interval: 10000 + +#避免由于task不能正常取消而使taskmanager服务挂掉 +task.cancellation.timeout: 0 \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/hbase/create_table b/TSG发布版本更新记录/TSG-21.11/hbase/create_table index 5eacc38..9423d61 100644 --- a/TSG发布版本更新记录/TSG-21.11/hbase/create_table +++ b/TSG发布版本更新记录/TSG-21.11/hbase/create_table @@ -1,9 +1,11 @@ create_namespace 'tsg_galaxy' - create 'tsg_galaxy:relation_account_framedip', {NAME => 'radius', VERSIONS => 1} - +create 'tsg_galaxy:relation_framedip_account', {NAME => 'radius', VERSIONS => 1} -- Phoenix创建视图 CREATE view "tsg"."report_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR); -CREATE view "tsg_galaxy"."relation_account_framedip" ( ROWKEY VARCHAR PRIMARY KEY, "radius"."framed_ip" VARCHAR, "radius"."last_found_time" UNSIGNED_LONG); \ No newline at end of file +CREATE view "tsg_galaxy"."relation_account_framedip" ( ROWKEY VARCHAR PRIMARY KEY, "radius"."framed_ip" VARCHAR, "radius"."last_found_time" UNSIGNED_LONG); + + + diff --git a/TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml b/TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml new file mode 100644 index 0000000..a3ed1f1 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml @@ -0,0 +1,12 @@ + + + + + phoenix.schema.isNamespaceMappingEnabled + true + + + phoenix.schema.mapSystemTablesToNamespace + true + + diff --git a/TSG发布版本更新记录/TSG-21.11/job/xxl_job.sql b/TSG发布版本更新记录/TSG-21.11/job/xxl_job.sql new file mode 100644 index 0000000..a9a8b5c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/job/xxl_job.sql @@ -0,0 +1,3 @@ +-- 优化了druid压缩任务，需要先删除xxl_job_info 中的任务 + +INSERT INTO `xxl_job_info` VALUES ('86', '6', '0 0 1 * * ?', 'Compact-With-Day-Job', '2021-07-21 14:13:57', '2021-11-22 06:56:01', 'galaxy', '', 'FAILOVER', 'druidCompactSegmentJobHandler', '{\r\n\"dataSource\":\"top_urls_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_website_domain_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_client_ip_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_server_ip_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_external_host_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_internal_host_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"top_user_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"security_event_hits_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"proxy_event_hits_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_protocol_stat_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_metrics_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_summary_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_top_destination_ip_metrics_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"traffic_app_stat_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n};\r\n{\r\n\"dataSource\":\"sys_storage_log\",\r\n\"internalOrNumOfHistory\":\"1\",\r\n\"segmentGranularity\":\"DAY\"\r\n}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2021-07-21 14:13:57', '', '1', '1637514000000', '1637600400000'); diff --git a/TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf b/TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf index 9d6abde..b878e29 100644 --- a/TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf +++ b/TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf @@ -1,28 +1,28 @@ worker_processes auto; - + events { worker_connections 4096; } - + http { - + include mime.types; default_type application/octet-stream; sendfile on; keepalive_timeout 65; keepalive_requests 10000; - vhost_traffic_status_zone; + vhost_traffic_status_zone; vhost_traffic_status_filter_by_host on; - # - client_max_body_size 1024M; #(设置客户端请求体最大值) - client_body_buffer_size 128k; #(配置请求体缓存区大小,) + client_max_body_size 1024M; + client_body_buffer_size 128k; access_log off; # 关闭access日志 - + + upstream qgwService { server 192.168.44.10:8183; server 192.168.44.13:8183; } - + upstream jobAdmin { server 192.168.44.10:8184; server 192.168.44.13:8184; @@ -31,6 +31,13 @@ http { upstream hos { server 192.168.44.10:8186; server 192.168.44.13:8186; + keepalive 1000; + keepalive_timeout 65; + keepalive_requests 100000; + } + geo $islocalip { + default 0; + #192.168.40.0/24 1; #添加本地不需要加密的ip，支持掩码形式，格式为xxx.xxx.xxx.xxx 1; } upstream nacos { server 192.168.44.11:8848; @@ -52,7 +59,7 @@ http { gzip_vary on; #是否传输gzip压缩标志 } } - + server { listen 8181; server_name localhost; @@ -67,17 +74,26 @@ http { vhost_traffic_status_display; vhost_traffic_status_display_format html; } - + } #hos非加密 server { listen 9098; server_name localhost; - proxy_set_header Host $http_host; client_max_body_size 1024m; location / { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $http_host; + set $ssl 1; #是否开启重定向加密，开启是1，关闭是0，默认关闭 + if ($islocalip = 0){ + set $ssl "${ssl}1"; + } if ($request_method = GET) { + set $ssl "${ssl}1"; + } + if ($ssl = "111") { return 302 https://$host:9097$request_uri; } proxy_pass http://hos$request_uri; diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml b/TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml new file mode 100644 index 0000000..bb99a3a --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml @@ -0,0 +1,18 @@ +# qgw 修改以下配置 + +spring: + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://{{ machine_host }}:3306/xxl_job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=GMT + username: {{ galaxy_mariadb_username }} + pin: {{ galaxy_mariadb_key }} + +hbase: + zookeeperQuorum: {{ machine_host }}:2181 + zookeeperZnodeParent: /hbase + rpcTimeout: 60000 + report: + dbname: tsg + tableName: report_result + columnFamily: response + columnName: result diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/active_defence_event.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/active_defence_event.json new file mode 100644 index 0000000..44aeb53 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/active_defence_event.json @@ -0,0 +1,322 @@ +{ + "type": "record", + "name": "active_defence_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "schema_query": { + "dimensions": [ + "common_policy_id", + "ad_target_ip", + "ad_cc_target_url" + ], + "metrics": [ + "ad_target_ip", + "ad_sent_byte_num", + "ad_sent_pkt_num", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num" + ], + "filters": [ + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_protocol", + "common_address_type", + "ad_sent_byte_num", + "ad_sent_pkt_num", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num" + ] + }, + "schema_type": { + "REFLECTION": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_target_ip", + "ad_target_port", + "ad_target_ip_location", + "ad_target_ip_asn", + "ad_reflector_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_reflector_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num" + ] + }, + "FLOOD": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_target_ip", + "ad_target_port", + "ad_target_ip_location", + "ad_target_ip_asn", + "ad_claimed_src_ip_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_claimed_src_ip_profile_id", + "ad_protocol" + ] + }, + "CC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_cc_target_url", + "ad_claimed_src_ip_profile_id", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_cc_target_url", + "ad_claimed_src_ip_profile_id", + "ad_protocol" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_cc_target_url" + ] + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ad_target_ip", + "label": "Target IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_ip_country,geo_asn", + "appendTo": "ad_target_ip_location,ad_target_ip_asn" + } + }, + "type": "string" + }, + { + "name": "ad_target_port", + "label": "Target Port", + "type": "int" + }, + { + "name": "ad_cc_target_url", + "label": "Target URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ad_target_ip_location", + "label": "Target Location", + "type": "string" + }, + { + "name": "ad_target_ip_asn", + "label": "Target ASN", + "type": "string" + }, + { + "name": "ad_protocol", + "label": "Protocol", + "type": "string" + }, + { + "name": "ad_method", + "label": "Method", + "type": "string" + }, + { + "name": "ad_claimed_src_ip_profile_id", + "label": "Claimed Profile ID", + "type": "int" + }, + { + "name": "ad_reflector_profile_id", + "label": "Reflector Profile ID", + "type": "int" + }, + { + "name": "ad_sent_pkt_num", + "label": "Packets Sent", + "type": "int" + }, + { + "name": "ad_sent_byte_num", + "label": "Bytes Sent", + "type": "int" + }, + { + "name": "ad_cc_initiate_connection_num", + "label": "Initiate Numbers", + "type": "int" + }, + { + "name": "ad_cc_established_connection_num", + "label": "Established Numbers", + "type": "int" + }, + { + "name": "ad_cc_rejected_connection_num", + "label": "Rejected Numbers", + "type": "int" + }, + { + "name": "ad_generate_time", + "label": "Generate Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "int" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-filter.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-filter.json new file mode 100644 index 0000000..a0a03b0 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-filter.json @@ -0,0 +1,99 @@ +{ + "version": "1.0", + "name": "ClickHouse-Raw", + "namespace": "ClickHouse", + "filters": [ + { + "name":"@start", + "value": "'2021-10-19 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-10-20 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='118.180.48.74'", + "common_client_ip='120.242.132.200'", + "common_internal_ip='223.116.37.192'", + "common_server_ip='8.8.8.8'", + "common_server_ip='114.114.114.114'", + "common_server_ip!='114.114.114.114'", + "common_server_ip='120.239.72.226'", + "common_external_ip='111.10.53.14'", + "common_client_port=52607", + "common_server_port=443", + "common_c2s_pkt_num>5", + "common_s2c_pkt_num>5", + "common_c2s_byte_num>100", + "common_s2c_byte_num<200", + "common_schema_type='DNS'", + "common_establish_latency_ms>200", + "common_con_duration_ms>10000", + "common_stream_trace_id=1153021139190754263", + "common_tcp_client_isn=2857077935", + "common_tcp_server_isn=0", + "http_domain='qq.com'", + "http_domain!='qq.com'", + "http_domain='yunser.com'", + "mail_account='abc@xx.com'", + "mail_subject='test'", + "dns_qname='qbwup.imtt.qq.com'", + "ssl_sni='mmbiz.qpic.cn'", + "ssl_sni='openai.qq.com'", + "ssl_con_latency_ms>100", + "ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'", + "common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'", + "common_server_ip='111.10.53.14' and common_server_port=443", + "common_server_ip like '120.239%'", + "common_server_ip not like '120.239%'", + "common_server_ip like '%114.114%'", + "mail_account like 'abc@%'", + "http_domain like '%baidu.com%'", + "ssl_sni like '%google.com'", + "http_domain like 'baidu%'", + "http_domain like '%baidu.com%'", + "common_client_ip in ('120.239.72.226','114.114.114.114')", + "common_client_ip not in ('120.239.72.226','114.114.114.114')", + "common_server_ip='116.177.248.126' and notEmpty(http_domain)", + "common_server_ip='116.177.248.126' and common_client_ip='120.242.132.200'", + "common_server_ip='116.177.248.126' and common_stream_trace_id=1153021139190754263", + "common_client_ip='120.242.132.200' and common_server_ip='116.177.248.126'", + "http_domain='qq.com' or common_server_ip='120.239.72.226'", + "common_server_port not in (80,443)", + "http_domain not like '%qq.com'" + ] + }, + { + "name":"@index_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='118.180.48.74'", + "common_client_ip='120.242.132.200'", + "common_server_ip='114.114.114.114'", + "common_server_ip!='114.114.114.114'", + "common_server_ip='120.239.72.226'", + "http_domain='qq.com'", + "http_domain!='qq.com'", + "http_domain='yunser.com'", + "ssl_sni='mmbiz.qpic.cn'", + "ssl_sni='openai.qq.com'", + "common_server_ip like '120.239%'", + "common_server_ip not like '120.239%'", + "common_server_ip like '%114.114%'", + "common_subscriber_id='%test%'", + "http_domain like 'baidu%'", + "http_domain like '%baidu.com%'", + "common_client_ip in ('120.239.72.226','114.114.114.114')", + "common_client_ip not in ('120.239.72.226','114.114.114.114')", + "common_server_ip='116.177.248.126' and notEmpty(http_domain)", + "common_server_ip='116.177.248.126' and common_client_ip='120.242.132.200'", + "common_server_ip='116.177.248.126' and common_stream_trace_id=1153021139190754263", + "common_client_ip='120.242.132.200' and common_server_ip='116.177.248.126'", + "http_domain='qq.com' or common_server_ip='120.239.72.226'" + ] + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-queries-template.sql b/TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-queries-template.sql new file mode 100644 index 0000000..fc08e8c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/ck-queries-template.sql @@ -0,0 +1,118 @@ +--Q01.Count(1) +select count(1) from session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) +--Q02.All Fields Query (default) +SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) LIMIT 30 +--Q03.All Fields Query order by Time desc +SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q04.All Fields Query order by Time asc +SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time asc LIMIT 30 +--Q05.All Fields Query by Filter +SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30 +--Q06.Default Fields Query by Filter +SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30 +--Q07.All Fields Query (sub query by time) +SELECT * FROM session_record AS session_record WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q08.All Fields Query (sub query by log id) +SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q09.Default Field Query (sub query by time) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q10.Default Field Query (sub query by log id) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( select common_log_id FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)) ORDER BY common_recv_time DESC LIMIT 30 +--Q11.Default Field Query by Server IP (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_common_server_ip AS session_record_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q12.Default Field Query by Client IP (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_common_client_ip AS session_record_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q13.Default Field Query by Domain (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_http_domain AS session_record_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q14.All Fields Query by Client IP (sub query by log id with index Table) +SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_common_client_ip AS session_record_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q15.All Fields Query by Server IP(sub query by log id with index Table) +SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_common_server_ip AS session_record_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q16.All Fields Query by Domain(sub query by log id with index Table) +SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_http_domain AS session_record_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q17.Session Logs Sent to Database Trend(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", count(common_log_id) AS "logs" FROM session_record AS session_record WHERE ( ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q18.Traffic Bandwidth Trend(Time Grain 30 second) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 30 SECOND)))) AS stat_time, sum(common_c2s_byte_num) AS bytes_sent, sum(common_s2c_byte_num) AS bytes_received, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets, sum(common_sessions) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time ORDER BY stat_time ASC LIMIT 10000 +--Q19.Log Tend by Type (Time Grain 5 minute) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) GROUP BY stat_time, common_schema_type ORDER BY stat_time ASC LIMIT 10000 +--Q20.Traffic Metrics Analytic +SELECT round(sum(common_s2c_byte_num) * 8 / 300,2) AS trafficInBits, round(sum(common_c2s_byte_num) * 8 / 300,2) AS trafficOutBits, round(sum(common_s2c_byte_num + common_c2s_byte_num) * 8 / 300,2) AS trafficTotalBits, round(sum(common_s2c_pkt_num) / 300,2) AS trafficInPackets, round(sum(common_c2s_pkt_num) / 300,2) AS trafficOutPackets, round(sum(common_s2c_pkt_num + common_c2s_pkt_num) / 300,2) AS trafficTotalPackets, round(sum(common_sessions) / 300,2) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) +--Q21.Traffic Endpoints Metrics Trend(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", uniq(common_internal_ip) AS "Unique Internal IP", uniq(common_external_ip) AS "Unique External IP", uniq(common_subscriber_id) AS "Unique Subscriber ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q22.Endpoint Unique Num by L4 Protocol +SELECT 'all' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) UNION ALL SELECT 'tcp' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) UNION ALL SELECT 'UDP' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_UDP', 'IPv6_UDP' ) +--Q23.One-sided Connection Trend(Time Grain 5 minute) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, (CASE WHEN common_stream_dir = 1 THEN 'c2s' WHEN common_stream_dir = 2 THEN 's2c' WHEN common_stream_dir = 3 THEN 'double' ELSE 'None' END) AS type, sum(common_sessions) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time, common_stream_dir ORDER BY stat_time ASC LIMIT 10000 +--Q24. Estimated One-sided Sessions with Bandwidth +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_sessions) AS "sessions", sum(if(common_stream_dir <> 3, common_sessions, 0)) AS "one_side_sessions", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", round(one_side_sessions / sessions, 2) AS one_side_percent FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q25.Estimated TCP Sequence Gap Loss +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", sum(common_c2s_tcp_lostlen + common_s2c_tcp_lostlen) AS "gap_loss_bytes", round(gap_loss_bytes / bytes, 2) AS gap_loss_percent FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q26.Top30 Server IP by Bytes +SELECT "server_ip" AS "server_ip" , SUM(coalesce("bytes",0)) AS "bytes" , SUM(coalesce("bytes_sent",0)) AS "Sent" , SUM(coalesce("bytes_received",0)) AS "Received" , SUM(coalesce("sessions",0)) AS "sessions" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(common_c2s_byte_num+common_s2c_byte_num) AS "bytes" , SUM(coalesce(common_sessions,0)) AS "sessions" , common_server_ip AS "server_ip" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" desc ) GROUP BY "server_ip" ORDER BY "bytes" desc LIMIT 30 +--Q27.Top30 Client IP by Sessions +SELECT common_client_ip , COUNT(*) AS sessions FROM session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_client_ip ORDER BY sessions desc LIMIT 0,30 +--Q28.Top30 TCP Server Ports by Sessions +SELECT "Server Port" AS "Server Port", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_port AS "Server Port", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Server Port" LIMIT 1048576) GROUP BY "Server Port" ORDER BY "Sessions" DESC LIMIT 30 +--Q29.Top30 Domian by Bytes +SELECT "domain" AS "Website Domain" , SUM(coalesce("bytes",0)) AS "Throughput" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "bytes" , http_domain AS "domain" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "domain" ORDER BY "bytes" desc ) GROUP BY "domain" ORDER BY "Throughput" desc LIMIT 30 +--Q30.Top30 Endpoint Devices by Bandwidth +SELECT "device_id" AS "device_id", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, common_device_id AS "device_id" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 30 +--Q31.Top30 Domain by Unique Client IP +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Client IP", 0)) AS "Client IP" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Client IP" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 30 +--Q32.Top100 Most Time Consuming Domains +SELECT "Domain" AS "Domain", avg(coalesce("Avg Establish Latency(ms)", 0)) AS "Avg Establish Latency(ms)" FROM (SELECT http_domain AS "Domain", avg(coalesce(common_establish_latency_ms, 0)) AS "Avg Establish Latency(ms)" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Avg Establish Latency(ms)" DESC LIMIT 100 +--Q33.Top30 Sources by Sessions +SELECT "source" AS "source", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(common_subscriber_id, ''), nullif(common_client_ip, '')) AS "source", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 30 +--Q34.Top30 Destinations by Sessions +SELECT "destination" AS "destination", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(http_domain, ''), nullif(common_server_ip, '')) AS "destination", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 30 +--Q35.Top30 Destination Regions by Bandwidth +SELECT "server_location" AS "server_location", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT arrayElement(splitByString(',', common_server_location), length(splitByString(',', common_server_location))) AS "server_location", sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "bytes", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 30 +--Q36.Top30 URLS by Sessions +SELECT "Http URL" AS "Http URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "Http URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http URL" LIMIT 1048576) GROUP BY "Http URL" ORDER BY "Sessions" DESC LIMIT 30 +--Q37.Top30 Destination Transmission APP by Bandwidth +SELECT "server_ip" AS "server_ip", groupUniqArray(coalesce("trans_app", 0)) AS "trans_app", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", groupUniqArray(concat(common_l4_protocol, '/', toString(common_server_port))) AS "trans_app", common_server_ip AS "server_ip" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 30 +--Q38.Browsing Users by Website domains and Sessions +SELECT "Subscriber ID" AS "Subscriber ID", "Http.Domain" AS "Http.Domain", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT http_domain AS "Http.Domain", common_subscriber_id AS "Subscriber ID", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) AND notEmpty(common_subscriber_id) ) GROUP BY "Http.Domain", "Subscriber ID" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Subscriber ID", "Http.Domain" ORDER BY "sessions" DESC LIMIT 10000 +--Q39.Top Domain and Server IP by Bytes Sent +SELECT "Http.Domain" AS "Http.Domain" , "Server IP" AS "Server IP" , SUM(coalesce("Bytes Sent",0)) AS "Bytes Sent" FROM ( SELECT common_server_ip AS "Server IP" , http_domain AS "Http.Domain" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "Bytes" , SUM(coalesce(common_c2s_byte_num,0)) AS "Bytes Sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "Bytes Received" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "Server IP" , "Http.Domain" ORDER BY "Bytes" desc LIMIT 1048576 ) GROUP BY "Http.Domain" , "Server IP" ORDER BY "Bytes Sent" desc LIMIT 10000 +--Q40.Top30 Website Domains by Client IP and Sessions +SELECT "Http.Domain" AS "Http.Domain", "Client IP" AS "Client IP", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT common_client_ip AS "Client IP", http_domain AS "Http.Domain", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Client IP", "Http.Domain" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Http.Domain", "Client IP" ORDER BY "sessions" DESC LIMIT 10000 +--Q41.Domain is Accessed by Unique Client IP Trend(bytes Time Grain 5 minute) +SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , http_domain AS Domain, COUNT(DISTINCT(common_client_ip)) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) AND http_domain IN ( SELECT http_domain FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY SUM(common_s2c_byte_num+common_c2s_byte_num) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , http_domain ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000 +--Q42. Domain is Accessed by Unique Client IP Trend(sessions,Time Grain 5 minute) +SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),3600)*3600) AS stat_time , http_domain , uniq (common_client_ip) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start)-604800 AND common_recv_time < toDateTime(@end) AND http_domain IN ( SELECT http_domain FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY COUNT(*) desc LIMIT 5 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 3600)*3600), http_domain ORDER BY stat_time desc LIMIT 10000 +--Q43.Bandwidth Trend with Device ID(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", common_device_id AS "Device ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time", "Device ID" LIMIT 10000 +--Q44.Internal IP by Sled IP and Sessions +SELECT "Internal IP" AS "Internal IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_internal_ip AS "Internal IP", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Sled IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 10000 +--Q45.Bandwidth Trend with Internal IP (Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_c2s_pkt_num + common_s2c_pkt_num, 0)) AS "Packets", sum(coalesce(common_sessions, 0)) AS "New Sessions", sum(coalesce(common_c2s_byte_num, 0)) AS "Bytes Sent", sum(coalesce(common_s2c_byte_num, 0)) AS "Bytes Received", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent", sum(coalesce(common_s2c_pkt_num, 0)) AS "Packets Received" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) GROUP BY "Receive Time" LIMIT 10000 +--Q46.Top30 Domains Detail with Internal IP +SELECT "Domain" AS "Domain", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_domain AS "Domain", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Sessions" DESC LIMIT 30 +--Q47.Top30 URLS Detail with Internal IP +SELECT "URL" AS "URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_url) ) GROUP BY "URL" LIMIT 1048576) GROUP BY "URL" ORDER BY "Sessions" DESC LIMIT 30 +--Q48.Top Domains with Unique Client IP and Subscriber ID +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Unique Client IP", 0)) AS "Unique Client IP", sum(coalesce("Unique Subscriber ID", 0)) AS "Unique Subscriber ID" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Unique Client IP", uniq(common_subscriber_id) AS "Unique Subscriber ID" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Unique Client IP" DESC LIMIT 100 +--Q49.Top100 Domains by Packets sent +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Packets Sent", 0)) AS "Packets Sent" FROM (SELECT http_domain AS "Http.Domain", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Packets Sent" DESC LIMIT 100 +--Q50.Internal and External asymmetric traffic +SELECT "Internal IP" AS "Internal IP", "External IP" AS "External IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_external_ip AS "External IP", common_internal_ip AS "Internal IP", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes Sent+Bytes Received", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Sled IP", "External IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "External IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 500 +--Q51.Client and Server ASN asymmetric traffic +SELECT "Client ASN" AS "Client ASN", "Server ASN" AS "Server ASN", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_asn AS "Server ASN", common_client_asn AS "Client ASN", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Server ASN", "Client ASN" LIMIT 1048576) GROUP BY "Client ASN", "Server ASN" ORDER BY "Sessions" DESC LIMIT 500 +--Q52.Top handshake latency by Website and Client IPs +SELECT "SSL.SNI" AS "SSL.SNI", "Client IP" AS "Client IP", avg(coalesce("Establish Latency(ms)", 0)) AS "Establish Latency(ms)" FROM (SELECT common_client_ip AS "Client IP", ssl_sni AS "SSL.SNI", avg(coalesce(common_establish_latency_ms, 0)) AS "Establish Latency(ms)" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Client IP", "SSL.SNI" LIMIT 1048576) GROUP BY "SSL.SNI", "Client IP" ORDER BY "Establish Latency(ms)" DESC LIMIT 500 +--Q53.Domain baidu.com Drill down Client IP +select common_client_ip as "Client IP" , avg(common_establish_latency_ms) as "Establishing Time Mean(ms)", count(1) as Responses,any(common_client_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Client IP" order by Responses desc limit 100 +--Q54.Domain baidu.com Drill down Server IP +select common_server_ip as "Server IP" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses,any(common_server_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Server IP" order by Responses desc limit 100 +--Q55.Domain baidu.com Drill down URI +select http_url as "URI" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "URI" order by Responses desc limit 100 +--Q56.L7 Protocol Metrics +select common_l7_protocol as "Protocol" , uniq(common_client_ip) as "Clients" , uniq(common_server_ip) as "Servers", count(1) as Sessions,sum(common_c2s_byte_num+common_s2c_byte_num) as bytes from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and notEmpty(common_l7_protocol) group by common_l7_protocol order by bytes desc +--Q57.L7 Protocol SIP Drill down Client IP +select common_client_ip as "Client IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_client_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Client IP" order by Sessions desc limit 100 +--Q58.L7 Protocol SIP Drill down Server IP +select common_server_ip as "Server IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_server_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Server IP" order by Sessions desc limit 100 +--Q59.Top5 Server IP keys with Unique Client IPs Trend (Grain 5 minute) +SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , common_server_ip AS server_ip, COUNT(DISTINCT(common_client_ip)) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_server_ip IN ( SELECT common_server_ip FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_server_ip ORDER BY count(*) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , server_ip ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000 \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/columns_cluster.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/columns_cluster.json new file mode 100644 index 0000000..d190d3c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/columns_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "columns_cluster", + "fields": [ + { + "name": "database", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/disks_cluster.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/disks_cluster.json new file mode 100644 index 0000000..70777c6 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/disks_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "disks_cluster", + "fields": [ + { + "name": "name", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/dos_event.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/dos_event.json new file mode 100644 index 0000000..cb22113 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/dos_event.json @@ -0,0 +1,348 @@ +{ + "type": "record", + "name": "dos_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "log_id", + "partition_key": "start_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + } + }, + "default_columns": [ + "log_id", + "attack_type", + "source_ip_list", + "destination_ip", + "severity", + "start_time", + "end_time", + "packet_rate", + "bit_rate", + "session_rate" + ] + }, + "fields": [ + { + "name": "start_time", + "label": "Start Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "attack_type", + "label": "Attack Type", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "TCP SYN Flood", + "value": "TCP SYN Flood" + }, + { + "code": "UDP Flood", + "value": "UDP Flood" + }, + { + "code": "ICMP Flood", + "value": "ICMP Flood" + }, + { + "code": "DNS Flood", + "value": "DNS Flood" + }, + { + "code": "DNS Amplification", + "value": "DNS Amplification" + } + ] + }, + "type": "string" + }, + { + "name": "severity", + "label": "Severity", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "Critical", + "value": "Critical" + }, + { + "code": "Severe", + "value": "Severe" + }, + { + "code": "Major", + "value": "Major" + }, + { + "code": "Warning", + "value": "Warning" + }, + { + "code": "Minor", + "value": "Minor" + } + ] + }, + "type": "string" + }, + { + "name": "conditions", + "label": "Conditions", + "type": "string" + }, + { + "name": "destination_ip", + "label": "Destination IP", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "destination_country", + "label": "Destination Country", + "type": "string" + }, + { + "name": "source_ip_list", + "label": "Source IPs", + "type": "string" + }, + { + "name": "source_country_list", + "label": "Source Countries", + "type": "string" + }, + { + "name": "session_rate", + "label": "Sessions/s", + "doc": { + "constraints": { + "type": "sessions/sec" + } + }, + "type": "long" + }, + { + "name": "packet_rate", + "label": "Packets/s", + "doc": { + "constraints": { + "type": "packets/sec" + } + }, + "type": "long" + }, + { + "name": "bit_rate", + "label": "Bits/s", + "doc": { + "constraints": { + "type": "bits/sec" + } + }, + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-filter.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-filter.json new file mode 100644 index 0000000..e8286b7 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-filter.json @@ -0,0 +1,21 @@ +{ + "version": "1.0", + "name": "druid-Raw", + "namespace": "druid", + "filters": [ + { + "name":"@start", + "value": "'2021-10-19 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-10-20 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_client_ip='192.168.44.21'and common_server_port=443" + ] + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-queries-template.sql b/TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-queries-template.sql new file mode 100644 index 0000000..c56d2c8 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/druid-queries-template.sql @@ -0,0 +1,92 @@ +--Q01.All Security Event Hits +select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end group by policy_id +--Q02.Security Event Hits with Policy ID 0 +select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id in (0) group by policy_id +--Q03.All Security Event Hits Trend by 5min A +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from security_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 10000 +--Q04.Security Event Hit Time（first and last time) A +select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from security_event_hits_log where policy_id in (0) group by policy_id +--Q05.Top 200 Security Policies +select policy_id, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP @start and __time =@start and __time <@end group by policy_id, action order by hits desc limit 200 +--Q07.All Proxy Event Hits +select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id +--Q08.Proxy Event Hits with Policy ID 0 +select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=0 group by policy_id +--Q09.All Proxy Event Hits Trend by 5min A +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time =TIMESTAMP @start and __time =@start and __time <@end group by policy_id, sub_action order by hits desc limit 200 +--Q13.Proxy Action Hits +select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by sub_action +--Q14.Proxy Action Hits Trend by 5min +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time = @start AND __time < @end UNION ALL SELECT sum(pinning_num) AS sessions, 'pinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(maybe_pinning_num) AS sessions, 'maybePinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end +--Q16.Traffic Metrics Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q17.Traffic Metrics Not Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(not_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time>= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q18.Traffic Metrics Maybe Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(maybe_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q19.Traffic Metrics Throughput Bytes IN/OUT +select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q23.Traffic Metrics Bandwidth Packets IN/OUT +select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q25.Traffic Metrics New and Live Sessions +select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q27.Traffic Metrics Security Throughput Bytes +select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end +--Q28.Traffic Metrics Security Throughput Packets +select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time = TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') +--Q31.Traffic Metrics Security Bandwidth Packets by 5Min +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100 +--Q35.Top 100 Internal IP by Sessions +select source as internal_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >=TIMESTAMP @start and __time = @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100 +--Q37.Top 100 Domain by Bytes +select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end group by policy_id +--Q41.Traffic Composition Metrics +SELECT APPROX_COUNT_DISTINCT_DS_HLL(ip_object) AS uniq_client_ip, SUM(one_sided_connections) AS one_sided_connections, SUM(uncategorized_bytes) AS total_uncategorized_bytes, SUM(fragmentation_packets) AS fragmentation_packets, SUM(sequence_gap_loss) AS sequence_gap_loss_bytes, SUM(s2c_byte_num+c2s_byte_num) AS summaryTotalBytes, SUM(s2c_pkt_num+c2s_pkt_num) AS summaryTotalPackets, SUM(sessions) AS summarySessions FROM traffic_summary_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end LIMIT 1 +--Q42.Traffic Composition Throughput +(SELECT SUM(c2s_byte_num + s2c_byte_num) as total_bytes, SUM(sessions) as total_sessions, (SUM(c2s_byte_num + s2c_byte_num) * 8)/((TIMESTAMP_TO_MILLIS(TIMESTAMP @end )-TIMESTAMP_TO_MILLIS(TIMESTAMP @start ))/1000) AS data_rate FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' LIMIT 1) UNION ALL ( SELECT SUM(sessions), 0, 0 FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' GROUP BY __time ORDER BY __time DESC LIMIT 1 ) +--Q43.Traffic Composition Protocol Tree +SELECT protocol_id, SUM(sessions) as sessions,SUM(c2s_byte_num) as c2s_byte_num, SUM(c2s_pkt_num) as c2s_pkt_num, SUM(s2c_byte_num) as s2c_byte_num, SUM(s2c_pkt_num) as s2c_pkt_num FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end GROUP BY protocol_id +--Q44.System Quota +SELECT log_type, SUM(used_size) as used_size, SUM(max_size) * 7/10 as max_size, TIME_FORMAT(LATEST(last_storage) * 1000,'YYYY-MM-dd') as first_storage FROM ( SELECT log_type, LATEST(used_size) as used_size, LATEST(max_size) as max_size, LATEST(last_storage) as last_storage FROM sys_storage_log WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '1' HOUR AND data_center != '' GROUP BY data_center,log_type ) GROUP BY log_type +--Q45.System Quota Daily Trend +select TIME_FORMAT(__time,'YYYY-MM-dd') as stat_time,log_type as type, sum(aggregate_size) as used_size from sys_storage_log where __time >= @start and __time < @end group by TIME_FORMAT(__time,'YYYY-MM-dd'), log_type +--Q46.Traffic Statistics(Metrics01) +select sum(total_hit_sessions) as total_hit_sessions, sum(total_bytes_transferred) as total_bytes_transferred, sum(total_packets_transferred) as total_packets_transferred, sum(total_new_sessions) as total_new_sessions , sum(total_close_sessions) as total_close_sessions, sum(average_new_sessions_per_second) as average_new_sessions_per_second , sum(average_bytes_per_second) as average_bytes_per_second , sum(average_packets_per_second) as average_packets_per_second , COUNT(DISTINCT(device_id)) as device_num, sum(live_sessions) as average_live_sessions from ( select device_id, sum(intercept_conn_num + monitor_conn_num + deny_conn_num + allow_conn_num) as total_hit_sessions, sum(total_in_bytes + total_out_bytes) as total_bytes_transferred, sum(total_in_packets + total_out_packets) as total_packets_transferred, sum(new_conn_num) as total_new_sessions, sum(close_conn_num) as total_close_sessions, avg(nullif(new_conn_num, 0))/ 5 as average_new_sessions_per_second, avg(nullif(total_in_bytes + total_out_bytes, 0))* 8 / 5 as average_bytes_per_second, avg(nullif(total_in_packets + total_out_packets, 0))/ 5 as average_packets_per_second, avg(nullif(established_conn_num, 0)) as live_sessions from traffic_metrics_log where __time >= @start and __time < @end group by device_id) \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-filter.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-filter.json new file mode 100644 index 0000000..525a02c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-filter.json @@ -0,0 +1,53 @@ +{ + "version": "1.0", + "name": "Engine-Raw", + "namespace": "Engine", + "filters": [ + { + "name":"@start", + "value": "'2021-10-19 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-10-20 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='36.189.226.21'", + "common_internal_ip='223.116.37.192'", + "common_server_ip='8.8.8.8'", + "common_external_ip='111.10.53.14'", + "common_client_port=52607", + "common_server_port=443", + "common_c2s_pkt_num>5", + "common_s2c_pkt_num>5", + "common_c2s_byte_num>100", + "common_s2c_byte_num<200", + "common_schema_type='DNS'", + "common_establish_latency_ms>200", + "common_con_duration_ms>10000", + "common_stream_trace_id=1153021139190754263", + "common_tcp_client_isn=2857077935", + "common_tcp_server_isn=0", + "http_domain='microsoft.com'", + "mail_account='abc@xx.com'", + "mail_subject='test'", + "dns_qname='qbwup.imtt.qq.com'", + "ssl_sni='note.youdao.com'", + "ssl_con_latency_ms>100", + "ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'", + "common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'", + "common_server_ip='111.10.53.14' and common_server_port=443", + "mail_account like 'abc@%'", + "http_domain like '%baidu.com%'", + "ssl_sni like '%youdao.com'", + "common_client_ip in ('36.189.226.21','111.10.53.14')", + "common_server_port not in (80,443)", + "notEmpty(http_domain)", + "http_domain not like '%microsoft.com'" + ] + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-queries-template.sql b/TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-queries-template.sql new file mode 100644 index 0000000..7e92090 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/engine-queries-template.sql @@ -0,0 +1,96 @@ +--Q01.CK DateTime +select toDateTime(common_recv_time) as common_recv_time from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 20 +--Q02.Standard DateTime +select FROM_UNIXTIME(common_recv_time) as common_recv_time from session_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) limit 20 +--Q03.count(1) +select count(1) from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) +--Q04.count(*) +select count(*) from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) +--Q05.UDF APPROX_COUNT_DISTINCT_DS_HLL +SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end and policy_id=0 group by policy_id +--Q06.UDF TIME_FLOOR_WITH_FILL +select TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','previous') as stat_time from session_record where common_recv_time > @start and common_recv_time < @end group by stat_time +--Q07.UDF GEO IP +select IP_TO_GEO(common_client_ip) as geo,IP_TO_CITY(common_server_ip) as city,IP_TO_COUNTRY(common_server_ip) as country from session_record limit 10 +--Q08.Special characters +select * from session_record where (common_protocol_label ='/$' or common_client_ip like'%') limit 10 +--Q09.Federation Query +select * from (select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','zero')) as stat_time from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by stat_time order by stat_time asc) +--Q10.Catalog Database +select * from tsg_galaxy_v3.session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 20 +--Q11.Session Record Logs +select * from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) AND @common_filter order by common_recv_time desc limit 20 +--Q12.Live Session Record Logs +select * from interim_session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) AND @common_filter order by common_recv_time desc limit 20 +--Q13.Transaction Record Logs +select * from transaction_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) order by common_recv_time desc limit 20 +--Q14.Security Event Logs +select * from security_event where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) AND @common_filter order by common_recv_time desc limit 0,20 +--Q15.Proxy Event Logs +select * from proxy_event where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q16.Radius Record Logs +select * from radius_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q17.GTPC Record Logs +select * from gtpc_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q18.Security Event Logs with fields +select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_log_id,common_policy_id,common_subscriber_id,common_client_ip,common_client_port,common_l4_protocol,common_address_type,common_server_ip,common_server_port,common_action,common_direction,common_sled_ip,common_client_location,common_client_asn,common_server_location,common_server_asn,common_c2s_pkt_num,common_s2c_pkt_num,common_c2s_byte_num,common_s2c_byte_num,common_schema_type,common_sub_action,common_device_id, FROM_UNIXTIME(common_start_time) as common_start_time, FROM_UNIXTIME(common_end_time) as common_end_time,common_establish_latency_ms,common_con_duration_ms,common_stream_dir,common_stream_trace_id,http_url,http_host,http_domain,http_request_body,http_response_body,http_cookie,http_referer,http_user_agent,http_content_length,http_content_type,http_set_cookie,http_version,http_response_latency_ms,http_action_file_size,http_session_duration_ms,mail_protocol_type,mail_account,mail_from_cmd,mail_to_cmd,mail_from,mail_to,mail_cc,mail_bcc,mail_subject,mail_attachment_name,mail_eml_file,dns_message_id,dns_qr,dns_opcode,dns_aa,dns_tc,dns_rd,dns_ra,dns_rcode,dns_qdcount,dns_ancount,dns_nscount,dns_arcount,dns_qname,dns_qtype,dns_qclass,dns_cname,dns_sub,dns_rr,ssl_sni,ssl_san,ssl_cn,ssl_pinningst,ssl_intercept_state,ssl_server_side_latency,ssl_client_side_latency,ssl_server_side_version,ssl_client_side_version,ssl_cert_verify,ssl_error,quic_version,quic_sni,quic_user_agent,ftp_account,ftp_url,ftp_content from security_event where common_recv_time >= @start and common_recv_time < @end order by common_recv_time desc limit 10000 +--Q19.Radius ON/OFF Logs For Frame IP +select framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >=toDateTime(@start) and event_timestamp = @start and event_timestamp < @end group by account +--Q21.Radius ON/OFF Logs total Account number +select count(distinct(framed_ip)) as active_ip_num , sum(acct_session_time) as online_duration from (select any(framed_ip) as framed_ip ,max(acct_session_time) as acct_session_time from radius_onff_log where account='000jS' and event_timestamp >= @start and event_timestamp < @end group by acct_session_id) +--Q22.Radius ON/OFF Logs Account Access Detail +select max(if(acct_status_type=1,event_timestamp,0)) as start_time,max(if(acct_status_type=2,event_timestamp,0)) as end_time, any(framed_ip) as ip,max(acct_session_time) as online_duration from radius_onff_log where event_timestamp >= @start and event_timestamp < @end group by acct_session_id order by start_time desc limit 200 +--Q23.Report for Client IP +select common_client_ip, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@end)) group by common_client_ip order by sessions desc limit 0,100 +--Q24.Report for Server IP +select common_server_ip, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by common_server_ip order by sessions desc limit 0,100 +--Q25.Report for SSL SNI +select ssl_sni, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by ssl_sni order by sessions desc limit 0,100 +--Q26.Report for SSL APP +select common_app_label as applicaiton, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by applicaiton order by sessions desc limit 0,100 +--Q27.Report for Domains +select http_domain AS domain,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(domain) GROUP BY domain ORDER BY bytes DESC LIMIT 100 +--Q28.Report for Domains with unique Client IP +select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_domain, uniq (common_client_ip) as nums from session_record where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_domain in (select http_domain from session_record where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_domain) group by http_domain order by SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_domain order by stat_time asc limit 500 +--Q29. Report for HTTP Host +SELECT http_host as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) GROUP BY host ORDER BY bytes DESC limit 100 union all SELECT 'totals' as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes, SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes, SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) +--Q30.Report for HTTP/HTTPS URLS with Sessions +SELECT http_url AS url,count(*) AS sessions FROM proxy_event WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) GROUP BY url ORDER BY sessions DESC LIMIT 100 +--Q31.Report for HTTP/HTTPS URLS with UNIQUE Client IP +select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_url, count(distinct(common_client_ip)) as nums from proxy_event where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_url IN (select http_url from proxy_event where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) group by http_url order by count(*) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_url order by stat_time asc limit 500 +--Q32.Report for Subscriber ID with Sessions +select common_subscriber_id as user, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) group by common_subscriber_id order by sessions desc limit 0,100 +--Q33.Report for Subscriber ID with Bandwidth +SELECT common_subscriber_id as user,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) GROUP BY user ORDER BY bytes DESC LIMIT 100 +--Q34.Report Unique Endpoints +select uniq(common_client_ip) as "Client IP",uniq(common_server_ip) as "Server IP",uniq(common_internal_ip) as "Internal IP",uniq(common_external_ip) as "External IP",uniq(http_domain) as "Domain",uniq(ssl_sni) as "SNI" from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) +--Q35.TopN Optimizer +SELECT http_url AS url, SUM(common_sessions) AS sessions FROM session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_url) GROUP BY http_url ORDER BY sessions DESC limit 10 +--Q36.All Security Event Hits Trend by 5min B +select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, sum(hits) as hits from security_event_hits_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') limit 10000 +--Q37.Security Event Hit Time（first and last time) B +select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from security_event_hits_log where policy_id in (0) group by policy_id +--Q38.All Proxy Event Hits Trend by 5min B +select FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end group by FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) limit 10000 +--Q39.Proxy Event Hit Time（first and last time) B +select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from proxy_event_hits_log where policy_id in (0) group by policy_id +--Q40.Traffic Composition Protocol Tree Trend +(SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes from traffic_protocol_stat_log where __time >= TIMESTAMP @start AND __time < TIMESTAMP @end and protocol_id = 'ETHERNET' group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id order by stat_time asc) union all (SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes from traffic_protocol_stat_log where __time >= TIMESTAMP @start AND __time < TIMESTAMP @end and protocol_id like CONCAT('ETHERNET','.%') and LENGTH(protocol_id) = LENGTH(REPLACE(protocol_id,'.','')) + 1 + 0 group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id order by stat_time asc) +--Q41.Traffic Metrics Security Action Hits Trend +select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) as statisticTime, sum(default_in_bytes + default_out_bytes) as default_bytes, sum(default_in_packets + default_out_packets) as default_packets, sum(default_conn_num) as default_sessions, sum(allow_in_bytes + allow_out_bytes) as allow_bytes, sum(allow_in_packets + allow_out_packets) as allow_packets, sum(allow_conn_num) as allow_sessions, sum(deny_in_bytes + deny_out_bytes) as deny_bytes, sum(deny_in_packets + deny_out_packets) as deny_packets, sum(deny_conn_num) as deny_sessions, sum(monitor_in_bytes + monitor_out_bytes) as monitor_bytes, sum(monitor_in_packets + monitor_out_packets) as monitor_packets, sum(monitor_conn_num) as monitor_sessions, sum(intercept_in_bytes + intercept_out_bytes) as intercept_bytes, sum(intercept_in_packets + intercept_out_packets) as intercept_packets, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >= @start and __time < @end group by FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) limit 100000 +--Q42.Traffic Metrics Proxy Action Hits Trend +SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) AS statisticTime,SUM(intcp_allow_num) AS intercept_allow_conn_num,SUM(intcp_mon_num) AS intercept_monitor_conn_num,SUM(intcp_deny_num) AS intercept_deny_conn_num,SUM(intcp_rdirt_num) AS intercept_redirect_conn_num,SUM(intcp_repl_num) AS intercept_replace_conn_num,SUM(intcp_hijk_num) AS intercept_hijack_conn_num,SUM(intcp_ins_num) AS intercept_insert_conn_num FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'PT1800S', 'zero')) LIMIT 100000 +--Q43.Traffic Statistics(Metrics02) +select FROM_UNIXTIME(stat_time) as max_active_date_by_sessions, total_live_sessions as max_live_sessions from ( select stat_time, sum(live_sessions) as total_live_sessions from ( select TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'P1D') as stat_time, device_id, avg(established_conn_num) as live_sessions from traffic_metrics_log where __time >= @start and __time<@end group by TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'P1D'), device_id) group by stat_time order by total_live_sessions desc limit 1 ) +--Q44.Traffic Summary(Bandwidth Trend) +select * from ( select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time,'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'), 'traffic_in_bytes' union all select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time,'traffic_out_bytes' as type,sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'),'traffic_out_bytes' ) order by stat_time asc limit 100000 +--Q45.Traffic Summary(Sessions Trend) +select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time, 'total_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'), 'total_conn_num' order by stat_time asc limit 10000 +--Q46.Domain Baidu.com Metrics +select FROM_UNIXTIME(min(common_recv_time)) as "First Seen" , FROM_UNIXTIME(max(common_recv_time)) as "Last Seen" , median(http_response_latency_ms) as "Server Processing Time Median(ms)", count(1) as Responses,any(common_server_location) as Location from session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND http_domain='baidu.com' +--Q47.TIME_FLOOR_WITH_FILL 01 +select "Device Group" as "Device Group" ,"Data Center" as "Data Center" ,FROM_UNIXTIME("End Time") as "End Time" , sum("counter") as "counter" from (select common_device_group as "Device Group" ,common_data_center as "Data Center" ,TIME_FLOOR_WITH_FILL (common_end_time,'PT1H','zero') as "End Time" ,count(common_log_id) as "counter" from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by "Device Group","Data Center","End Time") group by "Device Group" ,"Data Center" ,"End Time" order by "End Time" asc limit 5 +--Q48.TIME_FLOOR_WITH_FILL 02 +select FROM_UNIXTIME("End Time") as "End Time" , sum("counter") as "counter" from (select common_device_group as "Device Group" ,common_data_center as "Data Center" ,TIME_FLOOR_WITH_FILL (common_end_time,'PT1H','zero') as "End Time" ,count(common_log_id) as "counter" ,count(http_domain) as "HTTP.Domain" from security_event where ((common_recv_time >= toDateTime('2021-10-19 00:00:00') and common_recv_time < toDateTime('2021-10-20 00:00:00')) ) AND ( ( common_action = 2 ) ) group by "Device Group","Data Center","End Time") group by "End Time" order by "End Time" asc \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/gtpc_record.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/gtpc_record.json new file mode 100644 index 0000000..c769fa7 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/gtpc_record.json @@ -0,0 +1,1184 @@ +{ + "type": "record", + "name": "gtpc_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number", + "gtp_msg_type" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number", + "gtp_end_user_ipv4", + "gtp_end_user_ipv6", + "gtp_uplink_teid", + "gtp_downlink_teid", + "gtp_msg_type" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "GTP-C": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_app_surrogate_id", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number", + "gtp_end_user_ipv4", + "gtp_end_user_ipv6", + "gtp_uplink_teid", + "gtp_downlink_teid", + "gtp_msg_type" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "gtp_version", + "gtp_msg_type", + "gtp_imsi", + "gtp_imei", + "gtp_phone_number", + "common_client_ip", + "common_server_ip" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "gtp_version", + "gtp_msg_type", + "gtp_imsi", + "gtp_imei", + "gtp_phone_number", + "common_client_ip", + "common_server_ip" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_client_asn" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "GTP-C", + "value": "GTP-C" + } + ] + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "gtp_version", + "label": "Version", + "type": "string" + }, + { + "name": "gtp_apn", + "label": "APN", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + }, + { + "name": "gtp_msg_type", + "label": "Message Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "create", + "value": "create" + }, + { + "code": "modify", + "value": "modify" + }, + { + "code": "delete", + "value": "delete" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_end_user_ipv4", + "label": "End User Address V4", + "type": "string" + }, + { + "name": "gtp_end_user_ipv6", + "label": "End User Address V6", + "type": "string" + } + ] +} diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/interim_session_record.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/interim_session_record.json new file mode 100644 index 0000000..e25b36f --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/interim_session_record.json @@ -0,0 +1,2313 @@ +{ + "type": "record", + "name": "interim_session_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash", + "quic_sni", + "quic_version" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash", + "quic_sni" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash", + "quic_sni", + "quic_vesion" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "$ref": "public_schema_info.json#/schema_type/BASE" + }, + "HTTP": { + "$ref": "public_schema_info.json#/schema_type/HTTP" + }, + "MAIL": { + "$ref": "public_schema_info.json#/schema_type/MAIL" + }, + "DNS": { + "$ref": "public_schema_info.json#/schema_type/DNS" + }, + "SSL": { + "$ref": "public_schema_info.json#/schema_type/SSL" + }, + "QUIC": { + "$ref": "public_schema_info.json#/schema_type/QUIC" + }, + "FTP": { + "$ref": "public_schema_info.json#/schema_type/FTP" + }, + "BGP": { + "$ref": "public_schema_info.json#/schema_type/BGP" + }, + "APP": { + "$ref": "public_schema_info.json#/schema_type/APP" + }, + "SSH": { + "$ref": "public_schema_info.json#/schema_type/SSH" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "QUIC", + "value": "QUIC" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "APP", + "value": "APP" + }, + { + "code": "SSH", + "value": "SSH" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "http_url", + "label": "HTTP.URL", + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "label": "HTTP.Response Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content_length", + "label": "HTTP.Request Content Length", + "type": "string" + }, + { + "name": "http_request_content_type", + "label": "HTTP.Request Content Type", + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_content_length", + "label": "HTTP.Response Content Length", + "type": "string" + }, + { + "name": "http_response_content_type", + "label": "HTTP.Response Content Type", + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "mail_protocol_type", + "label": "Mail.Protocol Type", + "type": "string" + }, + { + "name": "mail_account", + "label": "Mail.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "mail_from_cmd", + "label": "Mail.From CMD", + "type": "string" + }, + { + "name": "mail_to_cmd", + "label": "Mail.To CMD", + "type": "string" + }, + { + "name": "mail_from", + "label": "Mail.From", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_to", + "label": "Mail.To", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_cc", + "label": "Mail.CC", + "type": "string" + }, + { + "name": "mail_bcc", + "label": "Mail.BCC", + "type": "string" + }, + { + "name": "mail_subject", + "label": "Mail.Subject", + "doc": { + "allow_query": "true", + "format": { + "functions": "decode_of_base64", + "param": "$.mail_subject_charset" + } + }, + "type": "string" + }, + { + "name": "mail_subject_charset", + "label": "Mail.Subject Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content", + "label": "Mail.Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content_charset", + "label": "Mail.Content Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_name", + "label": "Mail.Attachment", + "doc": { + "format": { + "functions": "decode_of_base64", + "param": "$.mail_attachment_name_charset" + } + }, + "type": "string" + }, + { + "name": "mail_attachment_name_charset", + "label": "Mail.Attachment Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_content", + "label": "Mail.Attachment Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_eml_file", + "label": "Mail.EML File", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "mail_snapshot", + "label": "Mail.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "dns_message_id", + "label": "DNS.Message ID", + "type": "int" + }, + { + "name": "dns_qr", + "label": "DNS.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_opcode", + "label": "DNS.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_aa", + "label": "DNS.AA", + "type": "int" + }, + { + "name": "dns_tc", + "label": "DNS.TC", + "type": "int" + }, + { + "name": "dns_rd", + "label": "DNS.RD", + "type": "int" + }, + { + "name": "dns_ra", + "label": "DNS.RA", + "type": "int" + }, + { + "name": "dns_rcode", + "label": "DNS.RCODE", + "type": "int" + }, + { + "name": "dns_qdcount", + "label": "DNS.QDCOUNT", + "type": "int" + }, + { + "name": "dns_ancount", + "label": "DNS.ANCOUNT", + "type": "int" + }, + { + "name": "dns_nscount", + "label": "DNS.NSCOUNT", + "type": "int" + }, + { + "name": "dns_arcount", + "label": "DNS.ARCOUNT", + "type": "int" + }, + { + "name": "dns_qname", + "label": "DNS.QNAME", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "dns_qtype", + "label": "DNS.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "dns_qclass", + "label": "DNS.QCLASS", + "type": "int" + }, + { + "name": "dns_cname", + "label": "DNS.CNAME", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "dns_sub", + "label": "DNS.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "dns_rr", + "label": "DNS.RR", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_version", + "label": "SSL.Version", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_sni", + "label": "SSL.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "ssl_san", + "label": "SSL.SAN", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_cn", + "label": "SSL.CN", + "type": "string" + }, + { + "name": "ssl_pinningst", + "label": "SSL.Pinning", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Not Pinning" + }, + { + "code": "1", + "value": "Pinning" + }, + { + "code": "2", + "value": "Maybe Pinning" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_intercept_state", + "label": "SSL.Intercept State", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Passthrough" + }, + { + "code": "1", + "value": "Intercept" + }, + { + "code": "2", + "value": "Shutdown" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_server_side_latency", + "label": "SSL.Server Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_client_side_latency", + "label": "SSL.Client Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_server_side_version", + "label": "SSL.Server Side Version", + "type": "string" + }, + { + "name": "ssl_client_side_version", + "label": "SSL.Client Side Version", + "type": "string" + }, + { + "name": "ssl_cert_verify", + "label": "SSL.Certificate Verify", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_error", + "label": "SSL.Error", + "type": "string" + }, + { + "name": "ssl_con_latency_ms", + "label": "SSL.Connection Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "ssl_ja3_fingerprint", + "label": "SSL.JA3", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ssl_ja3_hash", + "label": "SSL.JA3 hash", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ssl_cert_issuer", + "label": "SSL.Issuer", + "doc": { + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "ssl_cert_subject", + "label": "SSL.Subject", + "doc": { + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "quic_version", + "label": "QUIC.Version", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "quic_sni", + "label": "QUIC.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "quic_user_agent", + "label": "QUIC.User Agent", + "type": "string" + }, + { + "name": "ftp_account", + "label": "FTP.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ftp_url", + "label": "FTP.URL", + "type": "string" + }, + { + "name": "ftp_content", + "label": "FTP.Content", + "type": "string" + }, + { + "name": "ftp_link_type", + "label": "FTP.Link Type", + "type": "string" + }, + { + "name": "bgp_type", + "label": "BGP.Type", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "bgp_as_num", + "label": "BGP.AS Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "bgp_route", + "label": "BGP.Route", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_account", + "label": "VoIP.Calling Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_account", + "label": "VoIP.Called Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_number", + "label": "VoIP.Calling Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_number", + "label": "VoIP.Called Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_url", + "label": "Streaming.Media URL", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_protocol", + "label": "Streaming.Media Protocol", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "app_extra_info", + "label": "APP.Extra Info", + "type": "string" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "files" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ] + }, + "type": "int" + }, + { + "name": "ssh_version", + "label": "SSH.Version", + "type": "string" + }, + { + "name": "ssh_auth_success", + "label": "SSH.Authentication Result", + "type": "string" + }, + { + "name": "ssh_client_version", + "label": "SSH.Client Version", + "type": "string" + }, + { + "name": "ssh_server_version", + "label": "SSH.Server Version", + "type": "string" + }, + { + "name": "ssh_cipher_alg", + "label": "SSH.Encryption Algorithm", + "type": "string" + }, + { + "name": "ssh_mac_alg", + "label": "SSH.Signing Algorithm", + "type": "string" + }, + { + "name": "ssh_compression_alg", + "label": "SSH.Compression Algorithm", + "type": "string" + }, + { + "name": "ssh_kex_alg", + "label": "SSH. Key Exchange Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key_alg", + "label": "SSH.Server Host Key Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key", + "label": "SSH.Server Key Fingerprint", + "type": "string" + }, + { + "name": "ssh_hassh", + "label": "SSH.HASSH", + "type": "string" + } + ] +} diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_interim.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_interim.json new file mode 100644 index 0000000..0898ce1 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_interim.json @@ -0,0 +1,169 @@ +{ + "type": "record", + "name": "liveChart_interim", + "in": "INTERIM-SESSION-RECORD", + "out": "TRAFFIC-PROTOCOL-STAT", + "task": "Protocol-Distribution", + "doc": { + "timestamp": { + "name": "stat_time", + "type": "long" + }, + "dimensions": [ + { + "name": "protocol_id", + "fieldName": "common_protocol_label", + "type": "string" + }, + { + "name": "entrance_id", + "fieldName": "common_entrance_id", + "type": "string" + }, + { + "name": "isp", + "fieldName": "common_isp", + "type": "string" + }, + { + "name": "data_center", + "fieldName": "common_data_center", + "type": "string" + }, + { + "name": "device_group", + "fieldName": "common_device_group", + "type": "string" + } + ], + "metrics": [ + { + "function": "sum", + "name": "sessions", + "fieldName": "common_sessions", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_byte_num", + "fieldName": "common_c2s_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_byte_num", + "fieldName": "common_s2c_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_pkt_num", + "fieldName": "common_c2s_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_pkt_num", + "fieldName": "common_s2c_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_ipfrag_num", + "fieldName": "common_c2s_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_ipfrag_num", + "fieldName": "common_s2c_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_lostlen", + "fieldName": "common_c2s_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_lostlen", + "fieldName": "common_s2c_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_unorder_num", + "fieldName": "common_c2s_tcp_unorder_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_unorder_num", + "fieldName": "common_s2c_tcp_unorder_num", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_sip_num", + "fieldName": "common_server_ip", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_cip_num", + "fieldName": "common_client_ip", + "type": "long" + } + ], + "filters": [ + { + "fieldName": "common_protocol_label", + "type": "notempty" + } + ], + "transforms": [ + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_l7_protocol,." + }, + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_app_label,." + }, + { + "function": "flattenSpec", + "name": "data_center", + "fieldName": "common_device_tag", + "parameters": "$.tags[?(@.tag=='data_center')].value" + }, + { + "function": "flattenSpec", + "name": "device_group", + "fieldName": "common_device_tag", + "parameters": "$.tags[?(@.tag=='device_group')].value" + }, + { + "function": "hierarchy", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "." + } + ], + "action": [ + { + "label": "Default", + "metrics": "c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num" + } + ], + "granularity": { + "type": "period", + "period": "15S" + } + }, + "fields": [] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_session.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_session.json new file mode 100644 index 0000000..8a2c499 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/liveChart_session.json @@ -0,0 +1,169 @@ +{ + "type": "record", + "name": "liveChart_session", + "in": "SESSION-RECORD", + "out": "TRAFFIC-PROTOCOL-STAT", + "task": "Protocol-Distribution", + "doc": { + "timestamp": { + "name": "stat_time", + "type": "long" + }, + "dimensions": [ + { + "name": "protocol_id", + "fieldName": "common_protocol_label", + "type": "string" + }, + { + "name": "entrance_id", + "fieldName": "common_entrance_id", + "type": "string" + }, + { + "name": "isp", + "fieldName": "common_isp", + "type": "string" + }, + { + "name": "data_center", + "fieldName": "common_data_center", + "type": "string" + }, + { + "name": "device_group", + "fieldName": "common_device_group", + "type": "string" + } + ], + "metrics": [ + { + "function": "sum", + "name": "sessions", + "fieldName": "common_sessions", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_byte_num", + "fieldName": "common_c2s_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_byte_num", + "fieldName": "common_s2c_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_pkt_num", + "fieldName": "common_c2s_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_pkt_num", + "fieldName": "common_s2c_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_ipfrag_num", + "fieldName": "common_c2s_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_ipfrag_num", + "fieldName": "common_s2c_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_lostlen", + "fieldName": "common_c2s_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_lostlen", + "fieldName": "common_s2c_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_unorder_num", + "fieldName": "common_c2s_tcp_unorder_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_unorder_num", + "fieldName": "common_s2c_tcp_unorder_num", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_sip_num", + "fieldName": "common_server_ip", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_cip_num", + "fieldName": "common_client_ip", + "type": "long" + } + ], + "filters": [ + { + "fieldName": "common_protocol_label", + "type": "notempty" + } + ], + "transforms": [ + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_l7_protocol,." + }, + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_app_label,." + }, + { + "function": "flattenSpec", + "name": "data_center", + "fieldName": "common_device_tag", + "parameters": "$.tags[?(@.tag=='data_center')].value" + }, + { + "function": "flattenSpec", + "name": "device_group", + "fieldName": "common_device_tag", + "parameters": "$.tags[?(@.tag=='device_group')].value" + }, + { + "function": "hierarchy", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "." + } + ], + "action": [ + { + "label": "Default", + "metrics": "sessions,c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num,c2s_ipfrag_num,s2c_ipfrag_num,c2s_tcp_lostlen,s2c_tcp_lostlen,c2s_tcp_unorder_num,s2c_tcp_unorder_num" + } + ], + "granularity": { + "type": "period", + "period": "15S" + } + }, + "fields": [] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/meta_data.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/meta_data.json new file mode 100644 index 0000000..641d8e3 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/meta_data.json @@ -0,0 +1,87 @@ +{ + "metadata": [ + { + "namespace": "tsg_galaxy_v3", + "group": "CLICKHOUSE_GROUP", + "tables": [ + "radius_onff_log", + "session_record", + "session_record_common_client_ip", + "session_record_common_server_ip", + "session_record_http_domain", + "interim_session_record", + "transaction_record", + "radius_record", + "voip_record", + "gtpc_record", + "security_event", + "proxy_event", + "dos_event", + "active_defence_event", + "sys_packet_capture_event" + ] + }, + { + "namespace": "elasticsearch", + "group": "ELASTICSEARCH_GROUP", + "tables": [ + ] + }, + { + "namespace": "system", + "group": "CLICKHOUSE_GROUP", + "tables": [ + "query_log_cluster", + "tables_cluster", + "columns_cluster", + "disks_cluster", + "parts_cluster", + "processes", + "query_log" + ] + }, + { + "namespace": "druid", + "group": "DRUID_GROUP", + "tables": [ + "top_internal_host_log", + "top_website_domain_log", + "proxy_event_hits_log", + "sys_storage_log", + "security_event_hits_log", + "traffic_protocol_stat_log", + "top_server_ip_log", + "traffic_summary_log", + "traffic_metrics_log", + "top_user_log", + "top_urls_log", + "top_client_ip_log", + "top_external_host_log", + "traffic_app_stat_log", + "traffic_top_destination_ip_metrics_log" + ] + }, + { + "namespace": "etl", + "group": "ETL_GROUP", + "tables": [ + "liveChart_interim", + "liveChart_session" + ] + }, + { + "namespace":"tsg", + "group":"HBASE_GROUP", + "tables":[ + "report_result" + ] + }, + { + "namespace": "tsg_galaxy", + "group": "HBASE_GROUP", + "tables": [ + "relation_account_framedip" + ] + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/parts_cluster.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/parts_cluster.json new file mode 100644 index 0000000..c311abf --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/parts_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "parts_cluster", + "fields": [ + { + "name": "name", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/processes.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/processes.json new file mode 100644 index 0000000..75d74a9 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/processes.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "processes", + "fields": [ + { + "name": "query_id", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event.json new file mode 100644 index 0000000..a11f5c7 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event.json @@ -0,0 +1,1534 @@ +{ + "type": "record", + "name": "proxy_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_policy_id", + "common_sub_action", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "doh_host", + "doh_qname" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_sessions", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "doh_host", + "doh_qname" + ], + "filters": [ + "common_policy_id", + "common_sub_action", + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_l4_protocol", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_client_asn", + "common_server_asn", + "common_direction", + "common_schema_type", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "http_request_content_type", + "http_response_content_type", + "doh_host", + "doh_qname" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "HTTP": { + "$ref": "public_schema_info.json#/schema_type/HTTP" + }, + "DoH": { + "$ref": "public_schema_info.json#/schema_type/DoH" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_sub_action", + "common_schema_type" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "type": "int", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "48", + "value": "Manipulation" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "doc": { + "format": { + "functions": "set_value", + "param": "1" + } + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "DoH", + "value": "DoH" + } + ] + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "http_url", + "label": "HTTP.URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Response Line", + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Header", + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Header", + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_content_length", + "label": "HTTP.Request Content Length", + "type": "string" + }, + { + "name": "http_request_content_type", + "label": "HTTP.Request Content Type", + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_content_length", + "label": "HTTP.Response Content Length", + "type": "string" + }, + { + "name": "http_response_content_type", + "label": "HTTP.Response Content Type", + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "type": "long" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "doh_url", + "label": "DoH.URL", + "type": "string" + }, + { + "name": "doh_host", + "label": "DoH.Host", + "type": "string" + }, + { + "name": "doh_request_line", + "label": "DoH.Request Line", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "doh_response_line", + "doc": { + "visibility": "hidden" + }, + "label": "DoH.Response Line", + "type": "string" + }, + { + "name": "doh_cookie", + "label": "DoH.Cookie", + "type": "string" + }, + { + "name": "doh_referer", + "label": "DoH.Referer", + "type": "string" + }, + { + "name": "doh_user_agent", + "label": "DoH.User Agent", + "type": "string" + }, + { + "name": "doh_content_length", + "label": "DoH.Content Length", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "doh_content_type", + "label": "DoH.Content Type", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "doh_set_cookie", + "label": "DoH.Set Cookie", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "doh_version", + "label": "DoH.Version", + "type": "string" + }, + { + "name": "doh_message_id", + "label": "DoH.Message ID", + "type": "int" + }, + { + "name": "doh_qr", + "label": "DoH.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "REESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "doh_opcode", + "label": "DoH.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "doh_aa", + "label": "DoH.AA", + "type": "int" + }, + { + "name": "doh_tc", + "label": "DoH.TC", + "type": "int" + }, + { + "name": "doh_rd", + "label": "DoH.RD", + "type": "int" + }, + { + "name": "doh_ra", + "label": "DoH.RA", + "type": "int" + }, + { + "name": "doh_rcode", + "label": "DoH.RCODE", + "type": "int" + }, + { + "name": "doh_qdcount", + "label": "DoH.QDCOUNT", + "type": "int" + }, + { + "name": "doh_ancount", + "label": "DoH.ANCOUNT", + "type": "int" + }, + { + "name": "doh_nscount", + "label": "DoH.NSCOUNT", + "type": "int" + }, + { + "name": "doh_arcount", + "label": "DoH.ARCOUNT", + "type": "int" + }, + { + "name": "doh_qname", + "label": "DoH.QNAME", + "type": "string" + }, + { + "name": "doh_qtype", + "label": "DoH.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "doh_qclass", + "label": "DoH.QCLASS", + "type": "int" + }, + { + "name": "doh_cname", + "label": "DoH.CNAME", + "type": "string" + }, + { + "name": "doh_sub", + "label": "DoH.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "doh_rr", + "label": "DoH.RR", + "type": "string" + } + ] +} diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event_hits_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event_hits_log.json new file mode 100644 index 0000000..5e3ff8a --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/proxy_event_hits_log.json @@ -0,0 +1,58 @@ +{ + "type": "record", + "name": "proxy_event_hits_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "action", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "hits", + "type": "long" + }, + { + "name": "policy_id", + "type": "long" + }, + { + "name": "sub_action", + "type": "string" + }, + { + "name": "country", + "type": "string" + }, + { + "name": "location", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "ip_object", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/public_schema_info.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/public_schema_info.json new file mode 100644 index 0000000..a037a8b --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/public_schema_info.json @@ -0,0 +1,1797 @@ +{ + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + } + }, + "schema_type": { + "BASE": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "HTTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "http_url", + "http_host", + "http_domain", + "http_request_line", + "http_response_line", + "http_request_header", + "http_response_header", + "http_request_content", + "http_request_content_length", + "http_request_content_type", + "http_response_content", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_request_body_key", + "http_response_body_key", + "http_proxy_flag", + "http_sequence", + "http_snapshot", + "http_cookie", + "http_referer", + "http_user_agent", + "http_content_length", + "http_content_type", + "http_set_cookie", + "http_version", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "http_url", + "common_server_port", + "common_sub_action" + ] + }, + "MAIL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_content", + "mail_content_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_attachment_content", + "mail_eml_file", + "mail_snapshot" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "mail_from", + "mail_to", + "mail_subject" + ] + }, + "DNS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_client_ip", + "dns_qr", + "dns_qname", + "dns_qtype" + ] + }, + "SSL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_pinningst", + "ssl_intercept_state", + "ssl_server_side_latency", + "ssl_client_side_latency", + "ssl_server_side_version", + "ssl_client_side_version", + "ssl_cert_verify", + "ssl_error", + "ssl_con_latency_ms", + "ssl_ja3_fingerprint", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "ssl_sni", + "common_server_ip", + "common_server_port" + ] + }, + "QUIC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "quic_version", + "quic_sni", + "quic_user_agent" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "quic_sni", + "common_server_ip", + "common_server_port" + ] + }, + "FTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "ftp_account", + "ftp_url", + "ftp_content", + "ftp_link_type" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "ftp_url", + "common_server_ip", + "common_server_port" + ] + }, + "BGP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "bgp_type", + "bgp_as_num", + "bgp_route" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "bgp_type", + "bgp_as_num", + "common_server_ip", + "common_server_port" + ] + }, + "SIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port" + ] + }, + "RTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "APP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "app_extra_info" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_app_id", + "common_app_label", + "app_extra_info", + "common_server_ip", + "common_server_port" + ] + }, + "DoH": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "doh_url", + "doh_host", + "doh_request_line", + "doh_response_line", + "doh_cookie", + "doh_referer", + "doh_user_agent", + "doh_content_length", + "doh_content_type", + "doh_set_cookie", + "doh_version", + "doh_message_id", + "doh_qr", + "doh_opcode", + "doh_aa", + "doh_tc", + "doh_rd", + "doh_ra", + "doh_rcode", + "doh_qdcount", + "doh_ancount", + "doh_nscount", + "doh_arcount", + "doh_qname", + "doh_qtype", + "doh_qclass", + "doh_cname", + "doh_sub", + "doh_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_client_ip", + "doh_url", + "doh_qname", + "common_server_port" + ] + }, + "VoIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "SSH": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "ssh_auth_success" + ] + } + }, + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + }, + "fields": { + "common_encapsulation": { + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ] + }, + "common_has_dup_traffic": { + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log.json new file mode 100644 index 0000000..4f5e8d5 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "query_log", + "fields": [ + { + "name": "query_id", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log_cluster.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log_cluster.json new file mode 100644 index 0000000..d6e7583 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/query_log_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "query_log_cluster", + "fields": [ + { + "name": "type", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_onff_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_onff_log.json new file mode 100644 index 0000000..9201ebb --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_onff_log.json @@ -0,0 +1,37 @@ +{ + "type": "record", + "name": "radius_onff_log", + "namespace": "tsg_galaxy_v3", + "fields": [ + { + "name": "event_timestamp", + "label": "Event Time", + "type": "long" + }, + { + "name": "account", + "label": "Account", + "type": "string" + }, + { + "name": "framed_ip", + "label": "Framed IP", + "type": "string" + }, + { + "name": "acct_session_id", + "label": "Acct Session ID", + "type": "string" + }, + { + "name": "acct_status_type", + "label": "Acct Status Type", + "type": "int" + }, + { + "name": "acct_session_time", + "label": "Acct Session Time", + "type": "int" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_record.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_record.json new file mode 100644 index 0000000..19a20ff --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/radius_record.json @@ -0,0 +1,1376 @@ +{ + "type": "record", + "name": "radius_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "radius_nas_ip", + "radius_framed_ip", + "common_subscriber_id" + ], + "metrics": [ + "radius_framed_ip", + "radius_event_timestamp", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num" + ], + "filters": [ + "radius_framed_ip", + "common_subscriber_id", + "radius_packet_type", + "radius_acct_session_id", + "radius_acct_multi_session_id", + "radius_acct_status_type" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "RADIUS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "radius_packet_type", + "radius_nas_ip", + "radius_framed_ip", + "radius_account", + "radius_session_timeout", + "radius_idle_timeout", + "radius_acct_status_type", + "radius_acct_terminate_cause", + "radius_event_timestamp", + "radius_nas_port", + "radius_service_type", + "radius_framed_protocol", + "radius_callback_number", + "radius_callback_id", + "radius_termination_action", + "radius_called_station_id", + "radius_calling_station_id", + "radius_acct_delay_time", + "radius_acct_session_id", + "radius_acct_multi_session_id", + "radius_acct_input_octets", + "radius_acct_output_octets", + "radius_acct_input_packets", + "radius_acct_output_packets", + "radius_acct_session_time", + "radius_acct_link_count", + "radius_acct_interim_interval" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "radius_nas_ip", + "radius_framed_ip", + "radius_acct_status_type" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "radius_nas_ip", + "radius_framed_ip", + "radius_acct_status_type" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "48", + "value": "Manipulation" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "FTP", + "value": "FTP" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + }, + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "radius_packet_type", + "label": "Packet Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Access-Request" + }, + { + "code": "2", + "value": "Access-Accept" + }, + { + "code": "3", + "value": "Access-Reject" + }, + { + "code": "4", + "value": "Accounting-Request" + }, + { + "code": "5", + "value": "Accounting-Response" + }, + { + "code": "11", + "value": "Access-Challenge" + } + ] + }, + "type": "int" + }, + { + "name": "radius_account", + "label": "Account", + "doc": { + "format": { + "functions": "get_value", + "appendTo": "common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "radius_nas_ip", + "label": "Nas IP", + "type": "string" + }, + { + "name": "radius_framed_ip", + "label": "Framed IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "radius_session_timeout", + "label": "Session Timeout", + "type": "int" + }, + { + "name": "radius_idle_timeout", + "label": "Idle Timeout", + "type": "int" + }, + { + "name": "radius_acct_status_type", + "label": "ACC Status Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Start" + }, + { + "code": "2", + "value": "Stop" + }, + { + "code": "3", + "value": "Interim-Update" + }, + { + "code": "7", + "value": "Accounting-On" + }, + { + "code": "8", + "value": "Accounting-Off" + } + ] + }, + "type": "int" + }, + { + "name": "radius_acct_terminate_cause", + "label": "Acct Terminate Cause", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "User Request" + }, + { + "code": "2", + "value": "Lost Carrier" + }, + { + "code": "3", + "value": "Lost Service" + }, + { + "code": "4", + "value": "Idle Timeout" + }, + { + "code": "5", + "value": "Session Timeout" + }, + { + "code": "6", + "value": "Admin Reset" + }, + { + "code": "7", + "value": "Admin Reboot" + }, + { + "code": "8", + "value": "Port Error" + }, + { + "code": "9", + "value": "NAS Error" + }, + { + "code": "10", + "value": "NAS Request" + }, + { + "code": "11", + "value": "NAS Reboot" + }, + { + "code": "12", + "value": "Port Unneeded" + }, + { + "code": "13", + "value": "Port Preempted" + }, + { + "code": "14", + "value": "Port Suspended" + }, + { + "code": "15", + "value": "Service Unavailable" + }, + { + "code": "16", + "value": "Callback" + }, + { + "code": "17", + "value": "User Error" + }, + { + "code": "18", + "value": "Host Request" + } + ] + }, + "type": "int" + }, + { + "name": "radius_event_timestamp", + "label": "Event Timestamp", + "type": "int" + }, + { + "name": "radius_service_type", + "label": "Service Type", + "type": "int" + }, + { + "name": "radius_nas_port", + "label": "Nas Port", + "type": "int" + }, + { + "name": "radius_framed_protocol", + "label": "Framed Protocol", + "type": "int" + }, + { + "name": "radius_callback_number", + "label": "Callback Number", + "type": "string" + }, + { + "name": "radius_callback_id", + "label": "Callback ID", + "type": "string" + }, + { + "name": "radius_termination_action", + "label": "Termination Action", + "type": "int" + }, + { + "name": "radius_called_station_id", + "label": "Called Station ID", + "type": "string" + }, + { + "name": "radius_calling_station_id", + "label": "Calling Station ID", + "type": "string" + }, + { + "name": "radius_acct_delay_time", + "label": "Acct Delay Time", + "type": "int" + }, + { + "name": "radius_acct_session_id", + "label": "Acct Session ID", + "type": "string" + }, + { + "name": "radius_acct_multi_session_id", + "label": "Acct Multi Session ID", + "type": "string" + }, + { + "name": "radius_acct_input_octets", + "label": "Acct Input Octets", + "type": "long" + }, + { + "name": "radius_acct_output_octets", + "label": "Acct Output Octets", + "type": "long" + }, + { + "name": "radius_acct_input_packets", + "label": "Acct Input Packets", + "type": "long" + }, + { + "name": "radius_acct_output_packets", + "label": "Acct Output Packets", + "type": "long" + }, + { + "name": "radius_acct_session_time", + "label": "Acct Session Time", + "type": "int" + }, + { + "name": "radius_acct_link_count", + "label": "Acct Link Count", + "type": "int" + }, + { + "name": "radius_acct_interim_interval", + "label": "Acct Interim Interval", + "type": "int" + } + ] +} diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/relation_account_framedip.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/relation_account_framedip.json new file mode 100644 index 0000000..0ed7f7b --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/relation_account_framedip.json @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "relation_account_framedip", + "namespace": "tsg_galaxy", + "fields": [ + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/report_result.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/report_result.json new file mode 100644 index 0000000..7bc56e2 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/report_result.json @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "report_result", + "namespace": "tsg", + "fields": [ + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event.json new file mode 100644 index 0000000..084e1fb --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event.json @@ -0,0 +1,2424 @@ +{ + "type": "record", + "name": "security_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_policy_id", + "common_action", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_client_side_version", + "ssl_server_side_version", + "ssl_cert_issuer", + "ssl_cert_subject", + "mail_account", + "mail_from", + "mail_to", + "quic_sni", + "quic_version" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_mirrored_pkts", + "common_mirrored_bytes", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_client_side_latency", + "ssl_server_side_latency", + "ssl_cert_issuer", + "ssl_cert_subject", + "mail_account", + "mail_from", + "mail_to", + "quic_sni" + ], + "filters": [ + "common_policy_id", + "common_action", + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_mirrored_pkts", + "common_mirrored_bytes", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_direction", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "http_request_content_type", + "http_response_content_type", + "ssl_sni", + "ssl_ja3_hash", + "ssl_pinningst", + "ssl_intercept_state", + "ssl_client_side_version", + "ssl_server_side_version", + "ssl_cert_verify", + "ssl_client_side_latency", + "ssl_server_side_latency", + "ssl_cert_issuer", + "ssl_cert_subject", + "mail_account", + "mail_from", + "mail_to", + "mail_subject", + "quic_sni", + "quic_version" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "$ref": "public_schema_info.json#/schema_type/BASE" + }, + "HTTP": { + "$ref": "public_schema_info.json#/schema_type/HTTP" + }, + "MAIL": { + "$ref": "public_schema_info.json#/schema_type/MAIL" + }, + "DNS": { + "$ref": "public_schema_info.json#/schema_type/DNS" + }, + "SSL": { + "$ref": "public_schema_info.json#/schema_type/SSL" + }, + "QUIC": { + "$ref": "public_schema_info.json#/schema_type/QUIC" + }, + "FTP": { + "$ref": "public_schema_info.json#/schema_type/FTP" + }, + "BGP": { + "$ref": "public_schema_info.json#/schema_type/BGP" + }, + "SIP": { + "$ref": "public_schema_info.json#/schema_type/SIP" + }, + "RTP": { + "$ref": "public_schema_info.json#/schema_type/RTP" + }, + "APP": { + "$ref": "public_schema_info.json#/schema_type/APP" + }, + "SSH": { + "$ref": "public_schema_info.json#/schema_type/SSH" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "doc": { + "format": { + "functions": "set_value", + "param": "1" + } + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "QUIC", + "value": "QUIC" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + }, + { + "code": "APP", + "value": "APP" + }, + { + "code": "SSH", + "value": "SSH" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string" + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "allow_query": "true" + } + }, + { + "name": "http_url", + "label": "HTTP.URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "label": "HTTP.Response Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Header", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Header", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content_length", + "label": "HTTP.Request Content Length", + "type": "string" + }, + { + "name": "http_request_content_type", + "label": "HTTP.Request Content Type", + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_content_length", + "label": "HTTP.Response Content Length", + "type": "string" + }, + { + "name": "http_response_content_type", + "label": "HTTP.Response Content Type", + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "mail_protocol_type", + "label": "Mail.Protocol Type", + "type": "string" + }, + { + "name": "mail_account", + "label": "Mail.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "mail_from_cmd", + "label": "Mail.From CMD", + "type": "string" + }, + { + "name": "mail_to_cmd", + "label": "Mail.To CMD", + "type": "string" + }, + { + "name": "mail_from", + "label": "Mail.From", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_to", + "label": "Mail.To", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_cc", + "label": "Mail.CC", + "type": "string" + }, + { + "name": "mail_bcc", + "label": "Mail.BCC", + "type": "string" + }, + { + "name": "mail_subject", + "label": "Mail.Subject", + "doc": { + "allow_query": "true", + "format": { + "functions": "decode_of_base64", + "param": "$.mail_subject_charset" + } + }, + "type": "string" + }, + { + "name": "mail_subject_charset", + "label": "Mail.Subject Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content", + "label": "Mail.Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content_charset", + "label": "Mail.Content Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_name", + "label": "Mail.Attachment", + "doc": { + "format": { + "functions": "decode_of_base64", + "param": "$.mail_attachment_name_charset" + } + }, + "type": "string" + }, + { + "name": "mail_attachment_name_charset", + "label": "Mail.Attachment Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_content", + "label": "Mail.Attachment Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_eml_file", + "label": "Mail.EML File", + "doc": { + "constraints": { + "type": "file" + }, + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_snapshot", + "label": "Mail.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "dns_message_id", + "label": "DNS.Message ID", + "type": "int" + }, + { + "name": "dns_qr", + "label": "DNS.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_opcode", + "label": "DNS.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_aa", + "label": "DNS.AA", + "type": "int" + }, + { + "name": "dns_tc", + "label": "DNS.TC", + "type": "int" + }, + { + "name": "dns_rd", + "label": "DNS.RD", + "type": "int" + }, + { + "name": "dns_ra", + "label": "DNS.RA", + "type": "int" + }, + { + "name": "dns_rcode", + "label": "DNS.RCODE", + "type": "int" + }, + { + "name": "dns_qdcount", + "label": "DNS.QDCOUNT", + "type": "int" + }, + { + "name": "dns_ancount", + "label": "DNS.ANCOUNT", + "type": "int" + }, + { + "name": "dns_nscount", + "label": "DNS.NSCOUNT", + "type": "int" + }, + { + "name": "dns_arcount", + "label": "DNS.ARCOUNT", + "type": "int" + }, + { + "name": "dns_qname", + "label": "DNS.QNAME", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "dns_qtype", + "label": "DNS.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "dns_qclass", + "label": "DNS.QCLASS", + "type": "int" + }, + { + "name": "dns_cname", + "label": "DNS.CNAME", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "dns_sub", + "label": "DNS.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "dns_rr", + "label": "DNS.RR", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_version", + "label": "SSL.Version", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_sni", + "label": "SSL.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "ssl_san", + "label": "SSL.SAN", + "type": "string" + }, + { + "name": "ssl_cn", + "label": "SSL.CN", + "type": "string" + }, + { + "name": "ssl_pinningst", + "label": "SSL.Pinning", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Not Pinning" + }, + { + "code": "1", + "value": "Pinning" + }, + { + "code": "2", + "value": "Maybe Pinning" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_intercept_state", + "label": "SSL.Intercept State", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Passthrough" + }, + { + "code": "1", + "value": "Intercept" + }, + { + "code": "2", + "value": "Shutdown" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_server_side_latency", + "label": "SSL.Server Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_client_side_latency", + "label": "SSL.Client Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_server_side_version", + "label": "SSL.Server Side Version", + "type": "string" + }, + { + "name": "ssl_client_side_version", + "label": "SSL.Client Side Version", + "type": "string" + }, + { + "name": "ssl_cert_verify", + "label": "SSL.Certificate Verify", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_error", + "label": "SSL.Error", + "type": "string" + }, + { + "name": "ssl_con_latency_ms", + "label": "SSL.Connection Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "ssl_ja3_fingerprint", + "label": "SSL.JA3", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ssl_ja3_hash", + "label": "SSL.JA3 hash", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ssl_cert_issuer", + "label": "SSL.Issuer", + "doc": { + "allow_query": "true", + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "ssl_cert_subject", + "label": "SSL.Subject", + "doc": { + "allow_query": "true", + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "quic_version", + "label": "Quic.Version", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "quic_sni", + "label": "Quic.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "quic_user_agent", + "label": "Quic.User Agent", + "type": "string" + }, + { + "name": "ftp_account", + "label": "FTP.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ftp_url", + "label": "FTP.URL", + "type": "string" + }, + { + "name": "ftp_content", + "label": "FTP.Content", + "type": "string" + }, + { + "name": "ftp_link_type", + "label": "FTP.Link Type", + "type": "string" + }, + { + "name": "bgp_type", + "label": "BGP.Type", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "bgp_as_num", + "label": "BGP.AS Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "bgp_route", + "label": "BGP.Route", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_account", + "label": "VoIP.Calling Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_account", + "label": "VoIP.Called Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_number", + "label": "VoIP.Calling Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_number", + "label": "VoIP.Called Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_url", + "label": "Streaming.Media URL", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_protocol", + "label": "Streaming.Media Protocol", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "app_extra_info", + "label": "APP.Extra Info", + "type": "string" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "ssh_version", + "label": "SSH.Version", + "type": "string" + }, + { + "name": "ssh_auth_success", + "label": "SSH.Authentication Result", + "type": "string" + }, + { + "name": "ssh_client_version", + "label": "SSH.Client Version", + "type": "string" + }, + { + "name": "ssh_server_version", + "label": "SSH.Server Version", + "type": "string" + }, + { + "name": "ssh_cipher_alg", + "label": "SSH.Encryption Algorithm", + "type": "string" + }, + { + "name": "ssh_mac_alg", + "label": "SSH.Signing Algorithm", + "type": "string" + }, + { + "name": "ssh_compression_alg", + "label": "SSH.Compression Algorithm", + "type": "string" + }, + { + "name": "ssh_kex_alg", + "label": "SSH. Key Exchange Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key_alg", + "label": "SSH.Server Host Key Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key", + "label": "SSH.Server Key Fingerprint", + "type": "string" + }, + { + "name": "ssh_hassh", + "label": "SSH.HASSH", + "type": "string" + } + ] +} diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event_hits_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event_hits_log.json new file mode 100644 index 0000000..c445244 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/security_event_hits_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "security_event_hits_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "action", + "type": "long" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "hits", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "policy_id", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record.json new file mode 100644 index 0000000..9ca5c62 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record.json @@ -0,0 +1,2359 @@ +{ + "type": "record", + "name": "session_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "index_table": "session_record_common_client_ip,session_record_common_server_ip,session_record_http_domain", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "quic_sni", + "quic_version" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "quic_sni" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "quic_sni", + "quic_version" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "$ref": "public_schema_info.json#/schema_type/BASE" + }, + "HTTP": { + "$ref": "public_schema_info.json#/schema_type/HTTP" + }, + "MAIL": { + "$ref": "public_schema_info.json#/schema_type/MAIL" + }, + "DNS": { + "$ref": "public_schema_info.json#/schema_type/DNS" + }, + "SSL": { + "$ref": "public_schema_info.json#/schema_type/SSL" + }, + "QUIC": { + "$ref": "public_schema_info.json#/schema_type/QUIC" + }, + "FTP": { + "$ref": "public_schema_info.json#/schema_type/FTP" + }, + "BGP": { + "$ref": "public_schema_info.json#/schema_type/BGP" + }, + "APP": { + "$ref": "public_schema_info.json#/schema_type/APP" + }, + "SSH": { + "$ref": "public_schema_info.json#/schema_type/SSH" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "QUIC", + "value": "QUIC" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "APP", + "value": "APP" + }, + { + "code": "SSH", + "value": "SSH" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string" + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "http_url", + "label": "HTTP.URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "label": "HTTP.Response Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content_length", + "label": "HTTP.Request Content Length", + "type": "string" + }, + { + "name": "http_request_content_type", + "label": "HTTP.Request Content Type", + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_content_length", + "label": "HTTP.Response Content Length", + "type": "string" + }, + { + "name": "http_response_content_type", + "label": "HTTP.Response Content Type", + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "mail_protocol_type", + "label": "Mail.Protocol Type", + "type": "string" + }, + { + "name": "mail_account", + "label": "Mail.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "mail_from_cmd", + "label": "Mail.From CMD", + "type": "string" + }, + { + "name": "mail_to_cmd", + "label": "Mail.To CMD", + "type": "string" + }, + { + "name": "mail_from", + "label": "Mail.From", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_to", + "label": "Mail.To", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_cc", + "label": "Mail.CC", + "type": "string" + }, + { + "name": "mail_bcc", + "label": "Mail.BCC", + "type": "string" + }, + { + "name": "mail_subject", + "label": "Mail.Subject", + "doc": { + "allow_query": "true", + "format": { + "functions": "decode_of_base64", + "param": "$.mail_subject_charset" + } + }, + "type": "string" + }, + { + "name": "mail_subject_charset", + "label": "Mail.Subject Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content", + "label": "Mail.Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content_charset", + "label": "Mail.Content Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_name", + "label": "Mail.Attachment", + "doc": { + "format": { + "functions": "decode_of_base64", + "param": "$.mail_attachment_name_charset" + } + }, + "type": "string" + }, + { + "name": "mail_attachment_name_charset", + "label": "Mail.Attachment Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_content", + "label": "Mail.Attachment Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_eml_file", + "label": "Mail.EML File", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "mail_snapshot", + "label": "Mail.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "dns_message_id", + "label": "DNS.Message ID", + "type": "int" + }, + { + "name": "dns_qr", + "label": "DNS.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_opcode", + "label": "DNS.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_aa", + "label": "DNS.AA", + "type": "int" + }, + { + "name": "dns_tc", + "label": "DNS.TC", + "type": "int" + }, + { + "name": "dns_rd", + "label": "DNS.RD", + "type": "int" + }, + { + "name": "dns_ra", + "label": "DNS.RA", + "type": "int" + }, + { + "name": "dns_rcode", + "label": "DNS.RCODE", + "type": "int" + }, + { + "name": "dns_qdcount", + "label": "DNS.QDCOUNT", + "type": "int" + }, + { + "name": "dns_ancount", + "label": "DNS.ANCOUNT", + "type": "int" + }, + { + "name": "dns_nscount", + "label": "DNS.NSCOUNT", + "type": "int" + }, + { + "name": "dns_arcount", + "label": "DNS.ARCOUNT", + "type": "int" + }, + { + "name": "dns_qname", + "label": "DNS.QNAME", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "dns_qtype", + "label": "DNS.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "dns_qclass", + "label": "DNS.QCLASS", + "type": "int" + }, + { + "name": "dns_cname", + "label": "DNS.CNAME", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "dns_sub", + "label": "DNS.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "dns_rr", + "label": "DNS.RR", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_version", + "label": "SSL.Version", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_sni", + "label": "SSL.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "ssl_san", + "label": "SSL.SAN", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_cn", + "label": "SSL.CN", + "type": "string" + }, + { + "name": "ssl_pinningst", + "label": "SSL.Pinning", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Not Pinning" + }, + { + "code": "1", + "value": "Pinning" + }, + { + "code": "2", + "value": "Maybe Pinning" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_intercept_state", + "label": "SSL.Intercept State", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Passthrough" + }, + { + "code": "1", + "value": "Intercept" + }, + { + "code": "2", + "value": "Shutdown" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_server_side_latency", + "label": "SSL.Server Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_client_side_latency", + "label": "SSL.Client Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_server_side_version", + "label": "SSL.Server Side Version", + "type": "string" + }, + { + "name": "ssl_client_side_version", + "label": "SSL.Client Side Version", + "type": "string" + }, + { + "name": "ssl_cert_verify", + "label": "SSL.Certificate Verify", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_error", + "label": "SSL.Error", + "type": "string" + }, + { + "name": "ssl_con_latency_ms", + "label": "SSL.Connection Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "ssl_ja3_fingerprint", + "label": "SSL.JA3", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ssl_ja3_hash", + "label": "SSL.JA3 hash", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ssl_cert_issuer", + "label": "SSL.Issuer", + "doc": { + "allow_query": "true", + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "ssl_cert_subject", + "label": "SSL.Subject", + "doc": { + "allow_query": "true", + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "quic_version", + "label": "QUIC.Version", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "quic_sni", + "label": "QUIC.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "quic_user_agent", + "label": "QUIC.User Agent", + "type": "string" + }, + { + "name": "ftp_account", + "label": "FTP.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ftp_url", + "label": "FTP.URL", + "type": "string" + }, + { + "name": "ftp_content", + "label": "FTP.Content", + "type": "string" + }, + { + "name": "ftp_link_type", + "label": "FTP.Link Type", + "type": "string" + }, + { + "name": "bgp_type", + "label": "BGP.Type", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "bgp_as_num", + "label": "BGP.AS Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "bgp_route", + "label": "BGP.Route", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_account", + "label": "VoIP.Calling Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_account", + "label": "VoIP.Called Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_number", + "label": "VoIP.Calling Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_number", + "label": "VoIP.Called Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_url", + "label": "Streaming.Media URL", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_protocol", + "label": "Streaming.Media Protocol", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "app_extra_info", + "label": "APP.Extra Info", + "type": "string" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "files" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "ssh_version", + "label": "SSH.Version", + "type": "string" + }, + { + "name": "ssh_auth_success", + "label": "SSH.Authentication Result", + "type": "string" + }, + { + "name": "ssh_client_version", + "label": "SSH.Client Version", + "type": "string" + }, + { + "name": "ssh_server_version", + "label": "SSH.Server Version", + "type": "string" + }, + { + "name": "ssh_cipher_alg", + "label": "SSH.Encryption Algorithm", + "type": "string" + }, + { + "name": "ssh_mac_alg", + "label": "SSH.Signing Algorithm", + "type": "string" + }, + { + "name": "ssh_compression_alg", + "label": "SSH.Compression Algorithm", + "type": "string" + }, + { + "name": "ssh_kex_alg", + "label": "SSH. Key Exchange Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key_alg", + "label": "SSH.Server Host Key Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key", + "label": "SSH.Server Key Fingerprint", + "type": "string" + }, + { + "name": "ssh_hassh", + "label": "SSH.HASSH", + "type": "string" + } + ] +} diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_client_ip.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_client_ip.json new file mode 100644 index 0000000..9184e36 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_client_ip.json @@ -0,0 +1,71 @@ +{ + "type": "record", + "name": "session_record_common_client_ip", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "common_client_ip" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "common_schema_type", + "type": "string" + }, + { + "name": "common_client_port", + "type": "int" + }, + { + "name": "common_server_port", + "type": "int" + }, + { + "name": "common_app_label", + "type": "string" + }, + { + "name": "common_direction", + "type": "int" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_server_ip.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_server_ip.json new file mode 100644 index 0000000..a7c977f --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_common_server_ip.json @@ -0,0 +1,71 @@ +{ + "type": "record", + "name": "session_record_common_server_ip", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "common_server_ip" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "common_schema_type", + "type": "string" + }, + { + "name": "common_client_port", + "type": "int" + }, + { + "name": "common_server_port", + "type": "int" + }, + { + "name": "common_app_label", + "type": "string" + }, + { + "name": "common_direction", + "type": "int" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_http_domain.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_http_domain.json new file mode 100644 index 0000000..65414ea --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/session_record_http_domain.json @@ -0,0 +1,71 @@ +{ + "type": "record", + "name": "session_record_http_domain", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "http_domain" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "common_schema_type", + "type": "string" + }, + { + "name": "common_client_port", + "type": "int" + }, + { + "name": "common_server_port", + "type": "int" + }, + { + "name": "common_app_label", + "type": "string" + }, + { + "name": "common_direction", + "type": "int" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_packet_capture_event.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_packet_capture_event.json new file mode 100644 index 0000000..47879de --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_packet_capture_event.json @@ -0,0 +1,801 @@ +{ + "type": "record", + "name": "sys_packet_capture_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time" + }, + "fields": [ + { + "name": "common_recv_time", + "type": "long", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "label": "Receive Time" + }, + { + "name": "common_log_id", + "type": "long", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "label": "Log ID" + }, + { + "name": "common_policy_id", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Policy ID" + }, + { + "name": "common_subscriber_id", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Subscriber ID" + }, + { + "name": "common_imei", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "IMEI" + }, + { + "name": "common_imsi", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "IMSI" + }, + { + "name": "common_phone_number", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Phone Number" + }, + { + "name": "common_client_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Client IP" + }, + { + "name": "common_internal_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Internal IP" + }, + { + "name": "common_client_port", + "type": "int", + "label": "Client Port" + }, + { + "name": "common_l4_protocol", + "type": "string", + "label": "L4 Protocol" + }, + { + "name": "common_address_type", + "type": "int", + "doc": { + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "label": "Address Type" + }, + { + "name": "common_server_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Server IP" + }, + { + "name": "common_server_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Server Port" + }, + { + "name": "common_external_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "External IP" + }, + { + "name": "common_action", + "type": "int", + "doc": { + "allow_query": "true", + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "label": "Action" + }, + { + "name": "common_direction", + "type": "int", + "doc": { + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "label": "Direction" + }, + { + "name": "common_entrance_id", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Entrance ID" + }, + { + "name": "common_sled_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Sled IP" + }, + { + "name": "common_client_location", + "type": "string", + "label": "Client Location" + }, + { + "name": "common_client_asn", + "type": "string", + "label": "Client ASN" + }, + { + "name": "common_server_location", + "type": "string", + "label": "Server Location" + }, + { + "name": "common_server_asn", + "type": "string", + "label": "Server ASN" + }, + { + "name": "common_sessions", + "type": "long", + "label": "Sessions" + }, + { + "name": "common_c2s_pkt_num", + "type": "long", + "label": "Packets Sent" + }, + { + "name": "common_s2c_pkt_num", + "type": "long", + "label": "Packets Received" + }, + { + "name": "common_c2s_byte_num", + "type": "long", + "label": "Bytes Sent" + }, + { + "name": "common_s2c_byte_num", + "type": "long", + "label": "Bytes Received" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "type": "long" + }, + { + "name": "common_service", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Service" + }, + { + "name": "common_schema_type", + "type": "string", + "doc": { + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "FTP", + "value": "FTP" + } + ], + "visibility": "hidden" + }, + "label": "Schema Type" + }, + { + "name": "common_user_tags", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "User Tags" + }, + { + "name": "common_sub_action", + "type": "string", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "label": "Sub Action" + }, + { + "name": "common_user_region", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "User Region" + }, + { + "name": "common_device_id", + "type": "string", + "label": "Device ID" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "ISP" + }, + { + "name": "common_device_tag", + "type": "string", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "label": "Device Tag" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + } + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + } + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ] + }, + "label": "Encapsulation" + }, + { + "name": "common_app_label", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Application Label" + }, + { + "name": "common_tunnels", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Tunnels" + }, + { + "name": "common_protocol_label", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Protocol Label" + }, + { + "name": "common_app_id", + "type": "string", + "label": "Application ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "type": "string", + "label": "Surrogate ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "type": "string", + "label": "L7 Protocol" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "label": "Start Time" + }, + { + "name": "common_end_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "label": "End Time" + }, + { + "name": "common_establish_latency_ms", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Establish Latency(ms)" + }, + { + "name": "common_con_duration_ms", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Duration(ms)" + }, + { + "name": "common_stream_dir", + "type": "int", + "doc": { + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "label": "Stream Direction" + }, + { + "name": "common_address_list", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Address List" + }, + { + "name": "common_has_dup_traffic", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "label": "Duplication Traffic" + }, + { + "name": "common_stream_error", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Stream Error" + }, + { + "name": "common_stream_trace_id", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "Session ID" + }, + { + "name": "common_link_info_c2s", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(c2s)" + }, + { + "name": "common_link_info_s2c", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(s2c)" + }, + { + "name": "common_c2s_ipfrag_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Fragmentation Packets(c2s)" + }, + { + "name": "common_s2c_ipfrag_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Fragmentation Packets(s2c)" + }, + { + "name": "common_c2s_tcp_lostlen", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Sequence Gap Loss(c2s)" + }, + { + "name": "common_s2c_tcp_lostlen", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Sequence Gap Loss(s2c)" + }, + { + "name": "common_c2s_tcp_unorder_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Unorder Packets(c2s)" + }, + { + "name": "common_s2c_tcp_unorder_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Unorder Packets(s2c)" + }, + { + "name": "common_c2s_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(c2s)" + }, + { + "name": "common_s2c_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(s2c)" + }, + { + "name": "common_c2s_byte_retrans", + "type": "long", + "label": "Byte Retransmission(c2s)" + }, + { + "name": "common_s2c_byte_retrans", + "type": "long", + "label": "Byte Retransmission(s2c)" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "First TTL" + }, + { + "name": "common_processing_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "label": "Processing Time" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "nic_name", + "type": "string", + "label": "Nic Name" + }, + { + "name": "origin_source_mac", + "type": "string", + "label": "Origin Source Mac" + }, + { + "name": "origin_dest_mac", + "type": "string", + "label": "Origin Dest Mac" + }, + { + "name": "packet_url", + "type": "string", + "label": "Packet URL" + }, + { + "name": "pcap_storage_task_id", + "type": "int", + "label": "Task ID" + }, + { + "name": "pcap_storage_duration", + "type": "int", + "label": "Duration" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_storage_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_storage_log.json new file mode 100644 index 0000000..3bb3224 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/sys_storage_log.json @@ -0,0 +1,38 @@ +{ + "type": "record", + "name": "sys_storage_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "log_type", + "type": "string" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "max_size", + "type": "long" + }, + { + "name": "used_size", + "type": "long" + }, + { + "name": "aggregate_size", + "type": "long" + }, + { + "name": "last_storage", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/tables_cluster.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/tables_cluster.json new file mode 100644 index 0000000..4765d85 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/tables_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "tables_cluster", + "fields": [ + { + "name": "database", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_client_ip_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_client_ip_log.json new file mode 100644 index 0000000..78f3867 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_client_ip_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_client_ip_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "source", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_external_host_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_external_host_log.json new file mode 100644 index 0000000..68c229e --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_external_host_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_external_host_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "destination", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_internal_host_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_internal_host_log.json new file mode 100644 index 0000000..75347a5 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_internal_host_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_internal_host_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "source", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_server_ip_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_server_ip_log.json new file mode 100644 index 0000000..74258f1 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_server_ip_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_server_ip_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "destination", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_urls_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_urls_log.json new file mode 100644 index 0000000..7a0cc9b --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_urls_log.json @@ -0,0 +1,22 @@ +{ + "type": "record", + "name": "top_urls_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "url", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_user_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_user_log.json new file mode 100644 index 0000000..ebddb24 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_user_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_user_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "subscriber_id", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_website_domain_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_website_domain_log.json new file mode 100644 index 0000000..df86ea9 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/top_website_domain_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_website_domain_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "domain", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_app_stat_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_app_stat_log.json new file mode 100644 index 0000000..083ef7b --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_app_stat_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "traffic_app_stat_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "app_name", + "type": "string" + }, + { + "name": "sub_app_name", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_metrics_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_metrics_log.json new file mode 100644 index 0000000..6e0a690 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_metrics_log.json @@ -0,0 +1,214 @@ +{ + "type": "record", + "name": "traffic_metrics_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "allow_conn_num", + "type": "long" + }, + { + "name": "allow_in_bytes", + "type": "long" + }, + { + "name": "allow_in_packets", + "type": "long" + }, + { + "name": "allow_out_bytes", + "type": "long" + }, + { + "name": "allow_out_packets", + "type": "long" + }, + { + "name": "close_conn_num", + "type": "long" + }, + { + "name": "default_conn_num", + "type": "long" + }, + { + "name": "default_in_bytes", + "type": "long" + }, + { + "name": "default_in_packets", + "type": "long" + }, + { + "name": "default_out_bytes", + "type": "long" + }, + { + "name": "default_out_packets", + "type": "long" + }, + { + "name": "deny_conn_num", + "type": "long" + }, + { + "name": "deny_in_bytes", + "type": "long" + }, + { + "name": "deny_in_packets", + "type": "long" + }, + { + "name": "deny_out_bytes", + "type": "long" + }, + { + "name": "deny_out_packets", + "type": "long" + }, + { + "name": "device_id", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "intercept_conn_num", + "type": "long" + }, + { + "name": "intercept_in_bytes", + "type": "long" + }, + { + "name": "intercept_in_packets", + "type": "long" + }, + { + "name": "intercept_out_bytes", + "type": "long" + }, + { + "name": "intercept_out_packets", + "type": "long" + }, + { + "name": "established_conn_num", + "type": "long" + }, + { + "name": "monitor_conn_num", + "type": "long" + }, + { + "name": "monitor_in_bytes", + "type": "long" + }, + { + "name": "monitor_in_packets", + "type": "long" + }, + { + "name": "monitor_out_bytes", + "type": "long" + }, + { + "name": "monitor_out_packets", + "type": "long" + }, + { + "name": "new_conn_num", + "type": "long" + }, + { + "name": "total_in_bytes", + "type": "long" + }, + { + "name": "total_in_packets", + "type": "long" + }, + { + "name": "total_out_bytes", + "type": "long" + }, + { + "name": "total_out_packets", + "type": "long" + }, + { + "name": "alert_bytes", + "type": "long" + }, + { + "name": "hijk_bytes", + "type": "long" + }, + { + "name": "ins_bytes", + "type": "long" + }, + { + "name": "intcp_allow_num", + "type": "long" + }, + { + "name": "intcp_deny_num", + "type": "long" + }, + { + "name": "intcp_hijk_num", + "type": "long" + }, + { + "name": "intcp_ins_num", + "type": "long" + }, + { + "name": "intcp_mon_num", + "type": "long" + }, + { + "name": "intcp_rdirt_num", + "type": "long" + }, + { + "name": "intcp_repl_num", + "type": "long" + }, + { + "name": "maybe_pinning_num", + "type": "long" + }, + { + "name": "not_pinning_num", + "type": "long" + }, + { + "name": "pinning_num", + "type": "long" + }, + { + "name": "ad_cc_bytes", + "type": "long" + }, + { + "name": "ad_flood_bytes", + "type": "long" + }, + { + "name": "ad_reflection_bytes", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_protocol_stat_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_protocol_stat_log.json new file mode 100644 index 0000000..d37b603 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_protocol_stat_log.json @@ -0,0 +1,78 @@ +{ + "type": "record", + "name": "traffic_protocol_stat_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "protocol_id", + "type": "string" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "device_group", + "type": "string" + }, + { + "name": "sessions", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "c2s_ipfrag_num", + "type": "long" + }, + { + "name": "s2c_ipfrag_num", + "type": "long" + }, + { + "name": "c2s_tcp_lostlen", + "type": "long" + }, + { + "name": "s2c_tcp_lostlen", + "type": "long" + }, + { + "name": "c2s_tcp_unorder_num", + "type": "long" + }, + { + "name": "s2c_tcp_unorder_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_summary_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_summary_log.json new file mode 100644 index 0000000..26088b8 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_summary_log.json @@ -0,0 +1,78 @@ +{ + "type": "record", + "name": "traffic_summary_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "device_group", + "type": "string" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "schema_type", + "type": "string" + }, + { + "name": "ip_object", + "type": "string" + }, + { + "name": "sessions", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "one_sided_connections", + "type": "long" + }, + { + "name": "uncategorized_bytes", + "type": "long" + }, + { + "name": "fragmentation_packets", + "type": "long" + }, + { + "name": "sequence_gap_loss", + "type": "long" + }, + { + "name": "unorder_packets", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_top_destination_ip_metrics_log.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_top_destination_ip_metrics_log.json new file mode 100644 index 0000000..ece6294 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/traffic_top_destination_ip_metrics_log.json @@ -0,0 +1,46 @@ +{ + "type": "record", + "name": "traffic_top_destination_ip_metrics_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_data_center", + "type": "string" + }, + { + "name": "destination_ip", + "type": "string" + }, + { + "name": "attack_type", + "type": "string" + }, + { + "name": "session_rate", + "type": "long" + }, + { + "name": "packet_rate", + "type": "long" + }, + { + "name": "bit_rate", + "type": "long" + }, + { + "name": "partition_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/transaction_record.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/transaction_record.json new file mode 100644 index 0000000..f880bf7 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/transaction_record.json @@ -0,0 +1,1508 @@ +{ + "type": "record", + "name": "transaction_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_stream_trace_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "$ref": "public_schema_info.json#/schema_type/BASE" + }, + "HTTP": { + "$ref": "public_schema_info.json#/schema_type/HTTP" + }, + "MAIL": { + "$ref": "public_schema_info.json#/schema_type/MAIL" + }, + "DNS": { + "$ref": "public_schema_info.json#/schema_type/DNS" + }, + "SSL": { + "$ref": "public_schema_info.json#/schema_type/SSL" + }, + "QUIC": { + "$ref": "public_schema_info.json#/schema_type/QUIC" + }, + "FTP": { + "$ref": "public_schema_info.json#/schema_type/FTP" + }, + "BGP": { + "$ref": "public_schema_info.json#/schema_type/BGP" + }, + "SIP": { + "$ref": "public_schema_info.json#/schema_type/SIP" + }, + "RTP": { + "$ref": "public_schema_info.json#/schema_type/RTP" + }, + "APP": { + "$ref": "public_schema_info.json#/schema_type/APP" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "type": "long", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "label": "Receive Time" + }, + { + "name": "common_log_id", + "type": "long", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "label": "Log ID" + }, + { + "name": "common_policy_id", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Policy ID" + }, + { + "name": "common_subscriber_id", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Subscriber ID" + }, + { + "name": "common_imei", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "IMEI" + }, + { + "name": "common_imsi", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "IMSI" + }, + { + "name": "common_phone_number", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Phone Number" + }, + { + "name": "common_client_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "label": "Client IP" + }, + { + "name": "common_internal_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "label": "Internal IP" + }, + { + "name": "common_client_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Client Port" + }, + { + "name": "common_l4_protocol", + "type": "string", + "label": "L4 Protocol" + }, + { + "name": "common_address_type", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "label": "Address Type" + }, + { + "name": "common_server_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "label": "Server IP" + }, + { + "name": "common_server_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Server Port" + }, + { + "name": "common_external_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "label": "External IP" + }, + { + "name": "common_action", + "type": "int", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "label": "Action" + }, + { + "name": "common_direction", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "label": "Direction" + }, + { + "name": "common_entrance_id", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Entrance ID" + }, + { + "name": "common_sled_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Sled IP" + }, + { + "name": "common_client_location", + "type": "string", + "label": "Client Location" + }, + { + "name": "common_client_asn", + "type": "string", + "label": "Client ASN" + }, + { + "name": "common_server_location", + "type": "string", + "label": "Server Location" + }, + { + "name": "common_server_asn", + "type": "string", + "label": "Server ASN" + }, + { + "name": "common_sessions", + "type": "long", + "label": "Sessions" + }, + { + "name": "common_c2s_pkt_num", + "type": "long", + "label": "Packets Sent" + }, + { + "name": "common_s2c_pkt_num", + "type": "long", + "label": "Packets Received" + }, + { + "name": "common_c2s_byte_num", + "type": "long", + "label": "Bytes Sent" + }, + { + "name": "common_s2c_byte_num", + "type": "long", + "label": "Bytes Received" + }, + { + "name": "common_c2s_pkt_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Packets Sent(Diff)" + }, + { + "name": "common_s2c_pkt_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Packets Received(Diff)" + }, + { + "name": "common_c2s_byte_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Bytes Sent(Diff)" + }, + { + "name": "common_s2c_byte_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Bytes Received(Diff)" + }, + { + "name": "common_service", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Service" + }, + { + "name": "common_schema_type", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SIP", + "value": "SIP" + } + ], + "allow_query": "true" + }, + "label": "Schema Type" + }, + { + "name": "common_user_tags", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "User Tags" + }, + { + "name": "common_sub_action", + "type": "string", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "label": "Sub Action" + }, + { + "name": "common_user_region", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "User Region" + }, + { + "name": "common_device_id", + "type": "string", + "label": "Device ID" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "ISP" + }, + { + "name": "common_device_tag", + "type": "string", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "label": "Device Tag" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "label": "Encapsulation" + }, + { + "name": "common_app_label", + "type": "string", + "label": "Application Label" + }, + { + "name": "common_tunnels", + "type": "string", + "label": "Tunnels" + }, + { + "name": "common_protocol_label", + "type": "string", + "label": "Protocol Label" + }, + { + "name": "common_app_id", + "type": "string", + "label": "Application ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "type": "string", + "label": "Surrogate ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "type": "string", + "label": "L7 Protocol" + }, + { + "name": "common_service_category", + "type": { + "type": "array", + "items": "int" + }, + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "label": "FQDN Category" + }, + { + "name": "common_start_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "label": "Start Time" + }, + { + "name": "common_end_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "label": "End Time" + }, + { + "name": "common_establish_latency_ms", + "type": "long", + "label": "Establish Latency(ms)" + }, + { + "name": "common_con_duration_ms", + "type": "long", + "label": "Duration(ms)" + }, + { + "name": "common_stream_dir", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "label": "Stream Direction" + }, + { + "name": "common_address_list", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Address List" + }, + { + "name": "common_has_dup_traffic", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "label": "Duplication Traffic" + }, + { + "name": "common_stream_error", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Stream Error" + }, + { + "name": "common_stream_trace_id", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "Session ID" + }, + { + "name": "common_link_info_c2s", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(c2s)" + }, + { + "name": "common_link_info_s2c", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(s2c)" + }, + { + "name": "common_c2s_ipfrag_num", + "type": "long", + "label": "Fragmentation Packets(c2s)" + }, + { + "name": "common_s2c_ipfrag_num", + "type": "long", + "label": "Fragmentation Packets(s2c)" + }, + { + "name": "common_c2s_tcp_lostlen", + "type": "long", + "label": "Sequence Gap Loss(c2s)" + }, + { + "name": "common_s2c_tcp_lostlen", + "type": "long", + "label": "Sequence Gap Loss(s2c)" + }, + { + "name": "common_c2s_tcp_unorder_num", + "type": "long", + "label": "Unorder Packets(c2s)" + }, + { + "name": "common_s2c_tcp_unorder_num", + "type": "long", + "label": "Unorder Packets(s2c)" + }, + { + "name": "common_c2s_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(c2s)" + }, + { + "name": "common_s2c_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(s2c)" + }, + { + "name": "common_c2s_byte_retrans", + "type": "long", + "label": "Byte Retransmission(c2s)" + }, + { + "name": "common_s2c_byte_retrans", + "type": "long", + "label": "Byte Retransmission(s2c)" + }, + { + "name": "common_tcp_client_isn", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "TCP Client ISN" + }, + { + "name": "common_tcp_server_isn", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "TCP Server ISN" + }, + { + "name": "common_first_ttl", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "First TTL" + }, + { + "name": "common_processing_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "label": "Processing Time" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "http_url", + "type": "string", + "label": "HTTP.URL" + }, + { + "name": "http_host", + "type": "string", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "label": "HTTP.Host" + }, + { + "name": "http_domain", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "HTTP.Domain" + }, + { + "name": "http_request_line", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Request Line" + }, + { + "name": "http_response_line", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Response Line" + }, + { + "name": "http_request_header", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Request Headers" + }, + { + "name": "http_response_header", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Response Headers" + }, + { + "name": "http_request_content", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Request Content" + }, + { + "name": "http_request_content_length", + "label": "HTTP.Request Content Length", + "type": "string" + }, + { + "name": "http_request_content_type", + "label": "HTTP.Request Content Type", + "type": "string" + }, + { + "name": "http_response_content", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Response Content" + }, + { + "name": "http_response_content_length", + "label": "HTTP.Response Content Length", + "type": "string" + }, + { + "name": "http_response_content_type", + "label": "HTTP.Response Content Type", + "type": "string" + }, + { + "name": "http_request_body", + "type": "string", + "doc": { + "constraints": { + "type": "file" + } + }, + "label": "HTTP.Request Body" + }, + { + "name": "http_response_body", + "type": "string", + "doc": { + "constraints": { + "type": "file" + } + }, + "label": "HTTP.Response Body" + }, + { + "name": "http_request_body_key", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Request Body Key" + }, + { + "name": "http_response_body_key", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Response Body Key" + }, + { + "name": "http_proxy_flag", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Proxy Flag" + }, + { + "name": "http_sequence", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Sequence" + }, + { + "name": "http_snapshot", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Snapshot" + }, + { + "name": "http_cookie", + "type": "string", + "label": "HTTP.Cookie" + }, + { + "name": "http_referer", + "type": "string", + "label": "HTTP.Referer" + }, + { + "name": "http_user_agent", + "type": "string", + "label": "HTTP.User Agent" + }, + { + "name": "http_content_length", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Content Length" + }, + { + "name": "http_content_type", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Content Type" + }, + { + "name": "http_set_cookie", + "type": "string", + "label": "HTTP.Set Cookie" + }, + { + "name": "http_version", + "type": "string", + "label": "HTTP.Version" + }, + { + "name": "http_response_latency_ms", + "type": "long", + "label": "HTTP.Response Latency(ms)" + }, + { + "name": "http_session_duration_ms", + "type": "long", + "label": "HTTP.Session Duration(ms)" + }, + { + "name": "http_action_file_size", + "type": "int", + "label": "HTTP.Action File Size" + }, + { + "name": "dns_message_id", + "type": "int", + "label": "DNS.Message ID" + }, + { + "name": "dns_qr", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "label": "DNS.QR" + }, + { + "name": "dns_opcode", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "label": "DNS.OPCODE" + }, + { + "name": "dns_aa", + "type": "int", + "label": "DNS.AA" + }, + { + "name": "dns_tc", + "type": "int", + "label": "DNS.TC" + }, + { + "name": "dns_rd", + "type": "int", + "label": "DNS.RD" + }, + { + "name": "dns_ra", + "type": "int", + "label": "DNS.RA" + }, + { + "name": "dns_rcode", + "type": "int", + "label": "DNS.RCODE" + }, + { + "name": "dns_qdcount", + "type": "int", + "label": "DNS.QDCOUNT" + }, + { + "name": "dns_ancount", + "type": "int", + "label": "DNS.ANCOUNT" + }, + { + "name": "dns_nscount", + "type": "int", + "label": "DNS.NSCOUNT" + }, + { + "name": "dns_arcount", + "type": "int", + "label": "DNS.ARCOUNT" + }, + { + "name": "dns_qname", + "type": "string", + "label": "DNS.QNAME" + }, + { + "name": "dns_qtype", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "label": "DNS.QTYPE" + }, + { + "name": "dns_qclass", + "type": "int", + "label": "DNS.QCLASS" + }, + { + "name": "dns_cname", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "DNS.CNAME" + }, + { + "name": "dns_sub", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "label": "DNS.SUB" + }, + { + "name": "dns_rr", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "DNS.RR" + }, + { + "name": "sip_call_id", + "type": "string", + "label": "SIP.Call-ID" + }, + { + "name": "sip_originator_description", + "type": "string", + "label": "SIP.Originator" + }, + { + "name": "sip_responder_description", + "type": "string", + "label": "SIP.Responder" + }, + { + "name": "sip_user_agent", + "type": "string", + "label": "SIP.User-Agent" + }, + { + "name": "sip_server", + "type": "string", + "label": "SIP.Server" + }, + { + "name": "sip_originator_sdp_connect_ip", + "type": "string", + "label": "SIP.Originator IP" + }, + { + "name": "sip_originator_sdp_media_port", + "type": "int", + "label": "SIP.Originator Port" + }, + { + "name": "sip_originator_sdp_media_type", + "type": "string", + "label": "SIP.Originator Media Type" + }, + { + "name": "sip_originator_sdp_content", + "type": "string", + "label": "SIP.Originator Content" + }, + { + "name": "sip_responder_sdp_connect_ip", + "type": "string", + "label": "SIP.Responder IP" + }, + { + "name": "sip_responder_sdp_media_port", + "type": "int", + "label": "SIP.Responder Port" + }, + { + "name": "sip_responder_sdp_media_type", + "type": "string", + "label": "SIP.Responder Media Type" + }, + { + "name": "sip_responder_sdp_content", + "type": "string", + "label": "SIP.Responder Content" + }, + { + "name": "sip_duration", + "type": "int", + "label": "SIP.Duration" + }, + { + "name": "sip_bye", + "type": "string", + "label": "SIP.Bye" + } + ] +} diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/version.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/version.json new file mode 100644 index 0000000..19fd977 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/version.json @@ -0,0 +1,95 @@ +{ + "product": "185 Environment", + "version": "21.11", + "registered": "Geedge", + "updated": "2021-08-01 00:00:00", + "components": { + "oss": [ + { + "name": "zookeeper", + "version": "3.4.10", + "licenseType": "Apache License 2.0", + "description": "分布式应用程序协调服务" + }, + { + "name": "kafka", + "version": "2.11_1.0.0", + "licenseType": "Apache License 2.0", + "description": "消息队列" + }, + { + "name": "habse", + "version": "2.2.3", + "licenseType": "Apache License 2.0", + "description": "用于文件系统和存储Radius数据" + }, + { + "name": "flume", + "version": "1.9.0", + "licenseType": "Apache License 2.0", + "description": "日志补全传输" + }, + { + "name": "clickhouse", + "version": "20.3.12.112-cluster", + "licenseType": "Apache License 2.0", + "description": "原始日志数据库" + }, + { + "name": "druid", + "version": "0.18.1", + "licenseType": "Apache License 2.0", + "description": "分析实时数据并提供低延迟查询的OLAP应用程序" + }, + { + "name": "gohangout", + "version": "1.15.2.20210408", + "description": "动态获取原始日志表schema入库程序" + } + ], + "apps": [ + { + "name": "galaxy-qgw-service", + "version": "345", + "description": "数据平台对外统一查询网关" + }, + { + "name": "galaxy-report-service", + "version": "21.04.07", + "description": "自定义报表查询服务" + }, + { + "name": "galaxy-hos-service", + "version": "21.07.01", + "description": "对象存储服务" + }, + { + "name": "xxl-job-admin", + "version": "v1.3.20210408", + "description": "分布式任务调度平台" + }, + { + "name": "xxl-job", + "version": "v1.3.210413-rc1", + "description": "分布式任务调度平台-执行器" + } + ], + "tasks": [ + { + "name": "flume", + "version": "flume-config-20.08", + "description": "原始日志补全、subscriber更新、Radius上下线功能" + }, + { + "name": "druid", + "version": "druid-config-20.08", + "description": "所有分析日志任务" + }, + { + "name": "gohangout", + "version": "gohangout-config-20.08", + "description": "原始日志入库、上下线日志入库" + } + ] + } +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/schema/voip_record.json b/TSG发布版本更新记录/TSG-21.11/qgw/schema/voip_record.json new file mode 100644 index 0000000..833b41f --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/qgw/schema/voip_record.json @@ -0,0 +1,1377 @@ +{ + "type": "record", + "name": "voip_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "SIP": { + "$ref": "public_schema_info.json#/schema_type/SIP" + }, + "RTP": { + "$ref": "public_schema_info.json#/schema_type/RTP" + }, + "VoIP": { + "$ref": "public_schema_info.json#/schema_type/VoIP" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "VoIP", + "value": "VoIP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "files" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ] + }, + "type": "int" + } + ] +} diff --git a/TSG发布版本更新记录/TSG-21.11/record-21.11.txt b/TSG发布版本更新记录/TSG-21.11/record-21.11.txt new file mode 100644 index 0000000..434e006 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/record-21.11.txt @@ -0,0 +1,29 @@ +21.11更新记录 + +hbase +1.Phoenix集成，hbase新增表tsg_galaxy:relation_account_framedip，tsg_galaxy:relation_framedip_account +2.Phoenix创建视图tsg"."report_result，tsg_galaxy"."relation_account_framedip + +ck +1.新增字段 + +exporter +1. zk ,ck ,kafka,node-exporter + +预处理 + +1.更新jar包及配置 +2.新增了subid任务 + +网关 +更新镜像，schema + +调度任务 +1.更新了任务，sql执行器url添加/，更新了压缩任务 + +flink +1. 更新了checkpoint配置 + + +nginx +1. 更新了hos配置 \ No newline at end of file