From 351b5270f2484fcceffd0840d835df5177b30a38 Mon Sep 17 00:00:00 2001 From: zhanghongqing Date: Mon, 6 Dec 2021 16:25:54 +0800 Subject: [PATCH] 21.12 update --- TSG发布版本更新记录/TSG-21.12/docker/jdk.yml | 43 +- .../TSG-21.12/druid/traffic_app_stat_log.json | 2 +- .../config/RADIUS-RELATIONSHIP-HBASE-V2 | 24 + .../service_flow_config.properties | 24 + .../TSG-21.12/flink/account-framedip-Hbase/start.sh | 33 + .../TSG-21.12/flink/account-framedip-Hbase/stop.sh | 19 + .../TSG-21.12/flink/dos-detection/common.properties | 5 + .../TSG-21.12/flink/topN/kafka-flinksql-top.sql | 415 +++ .../TSG-21.12/flink/vpn-recommend/common.properties | 38 + .../flink/vpn-recommend/config/RECOMMENDATION-APP-CIP | 38 + .../TSG-21.12/flink/vpn-recommend/start.sh | 33 + .../TSG-21.12/flink/vpn-recommend/stop.sh | 19 + .../TSG-21.12/qgw/schema/dos_event.json | 353 +++ .../TSG-21.12/qgw/schema/engine-queries-template.sql | 116 + .../TSG-21.12/qgw/schema/gtpc_record.json | 1191 ++++++++ .../TSG-21.12/qgw/schema/hbase-filter.json | 15 + .../TSG-21.12/qgw/schema/hbase-queries-template.sql | 4 + .../TSG-21.12/qgw/schema/interim_session_record.json | 2318 ++++++++++++++++ .../TSG-21.12/qgw/schema/meta_data.json | 89 + .../TSG-21.12/qgw/schema/proxy_event.json | 1539 +++++++++++ .../TSG-21.12/qgw/schema/public_schema_info.json | 1976 ++++++++++++++ .../TSG-21.12/qgw/schema/radius_record.json | 1270 +++++++++ .../TSG-21.12/qgw/schema/recommendation_app_cip.json | 7 + .../TSG-21.12/qgw/schema/security_event.json | 2429 +++++++++++++++++ .../TSG-21.12/qgw/schema/session_record.json | 2364 ++++++++++++++++ .../TSG-21.12/qgw/schema/transaction_record.json | 1515 ++++++++++ .../TSG-21.12/qgw/schema/voip_record.json | 1384 ++++++++++ 27 files changed, 17220 insertions(+), 43 deletions(-) create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/start.sh create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/stop.sh create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/topN/kafka-flinksql-top.sql create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/config/RECOMMENDATION-APP-CIP create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/start.sh create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/stop.sh create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/dos_event.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/engine-queries-template.sql create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/gtpc_record.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-filter.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-queries-template.sql create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/interim_session_record.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/meta_data.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/proxy_event.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/public_schema_info.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/radius_record.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/recommendation_app_cip.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/security_event.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/session_record.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/transaction_record.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/schema/voip_record.json diff --git a/TSG发布版本更新记录/TSG-21.12/docker/jdk.yml b/TSG发布版本更新记录/TSG-21.12/docker/jdk.yml index 3061dd2..e76a615 100644 --- a/TSG发布版本更新记录/TSG-21.12/docker/jdk.yml +++ b/TSG发布版本更新记录/TSG-21.12/docker/jdk.yml @@ -1,42 +1 @@ -#1.指定基础镜像，并且必须是第一条指令 -#FROM alpine:latest -FROM alpine:3.15.0 - -#2.指明该镜像的作者和其电子邮件 -MAINTAINER galaxy - -#3.在构建镜像时，指定镜像的工作目录，之后的命令都是基于此工作目录，如果不存在，则会创建目录 -WORKDIR /opt/jdk1.8.0_73 - -#4.将一些安装包复制到镜像中，语法：ADD/COPY ... -## ADD与COPY的区别：ADD复制并解压，COPY仅复制 -## 注意~~~上传的瘦身后的jre -COPY jdk1.8.0_73 /opt/jdk1.8.0_73 -## glibc安装包如果从网络下载速度实在是太慢了，先提前下载复制到镜像中 -#COPY glibc-2.29-r0.apk /opt/jdk/ -#COPY glibc-bin-2.29-r0.apk /opt/jdk/ -#COPY glibc-i18n-2.29-r0.apk /opt/jdk/ - -#5.更新Alpine的软件源为阿里云，因为从默认官源拉取实在太慢了 -#RUN echo http://mirrors.aliyun.com/alpine/v3.10/main/ > /etc/apk/repositories && \ -# echo http://mirrors.aliyun.com/alpine/v3.10/community/ >> /etc/apk/repositories -#RUN apk update && apk upgrade - -#6.运行指定的命令 -## Alpine linux为了精简本身并没有安装太多的常用软件,apk类似于ubuntu的apt-get， -## 用来安装一些常用软V件，其语法如下：apk add bash wget curl git make vim docker -## wget是linux下的ftp/http传输工具，没安装会报错“/bin/sh: 　　wget: not found”，网上例子少安装wget -## ca-certificates证书服务，是安装glibc前置依赖 -RUN apk --no-cache add ca-certificates wget \ - && wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \ - && apk add glibc-2.29-r0.apk glibc-bin-2.29-r0.apk glibc-i18n-2.29-r0.apk \ - && rm -rf /var/cache/apk/* glibc-2.29-r0.apk glibc-bin-2.29-r0.apk glibc-i18n-2.29-r0.apk - -#7.配置环境变量 -## 注意~~~没有jdk啦，直接指向jre -ENV JAVA_HOME=/opt/jdk1.8.0_73 -ENV PATH=$JAVA_HOME/bin:$PATH -ENV CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar - -#容器启动时需要执行的命令 -#CMD ["java","-version"] \ No newline at end of file +jdk1.8.0_202 \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/druid/traffic_app_stat_log.json b/TSG发布版本更新记录/TSG-21.12/druid/traffic_app_stat_log.json index df52f6b..1da5ba3 100644 --- a/TSG发布版本更新记录/TSG-21.12/druid/traffic_app_stat_log.json +++ b/TSG发布版本更新记录/TSG-21.12/druid/traffic_app_stat_log.json @@ -52,7 +52,7 @@ "completionTimeout": "PT30M", "earlyMessageRejectionPeriod": "PT6H", "consumerProperties": { - "bootstrap.servers": "192.168.44.12:9094", + "bootstrap.servers": "kafkabootstrap", "sasl.mechanism": "PLAIN", "security.protocol": "SASL_PLAINTEXT", "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" diff --git a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 new file mode 100644 index 0000000..17dc226 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 @@ -0,0 +1,24 @@ +#管理kafka地址 +input.kafka.servers=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094 +#input.kafka.servers=192.168.44.12:9094 + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers=192.168.44.11,192.168.44.14,192.168.44.15 +#hbase.zookeeper.servers=192.168.44.11:2181 + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +input.kafka.topic=RADIUS-RECORD + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=radius-flink-20211124 + +#--------------------------------topology配置------------------------------# +#ip-account对应关系表 +hbase.framedip.table.name=tsg_galaxy:relation_framedip_account + +#定位库地址 +tools.library=/home/bigdata/topology/dat/ +#account-ip对应关系表 +hbase.account.table.name=tsg_galaxy:relation_account_framedip diff --git a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties new file mode 100644 index 0000000..17dc226 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties @@ -0,0 +1,24 @@ +#管理kafka地址 +input.kafka.servers=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094 +#input.kafka.servers=192.168.44.12:9094 + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers=192.168.44.11,192.168.44.14,192.168.44.15 +#hbase.zookeeper.servers=192.168.44.11:2181 + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +input.kafka.topic=RADIUS-RECORD + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=radius-flink-20211124 + +#--------------------------------topology配置------------------------------# +#ip-account对应关系表 +hbase.framedip.table.name=tsg_galaxy:relation_framedip_account + +#定位库地址 +tools.library=/home/bigdata/topology/dat/ +#account-ip对应关系表 +hbase.account.table.name=tsg_galaxy:relation_account_framedip diff --git a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/start.sh b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/start.sh new file mode 100644 index 0000000..ec2226f --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/start.sh @@ -0,0 +1,33 @@ +#! /bin/bash +#启动storm任务脚本 +source /etc/profile +#任务jar所在目录 +BASE_DIR=`pwd` +#jar name +JAR_NAME='radius-relation-21-12-06.jar' + +#cd $BASE_DIR +jar -xvf $BASE_DIR/$JAR_NAME service_flow_config.properties +function read_dir(){ + for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令 + do + if [ -d $1"/"$file ] #注意此处之间一定要加上空格，否则会报错 + then + read_dir $1"/"$file + else + num=`flink list | grep "$file" | wc -l` + if [ $num -eq "0" ];then + cat $1$file > $BASE_DIR/service_flow_config.properties + jar -uvf $BASE_DIR/$JAR_NAME service_flow_config.properties + flink run -d -p 1 $JAR_NAME + fi + fi +done +} +if [ $# != 1 ];then + echo "usage: ./startall.sh [Configuration path]" + exit 1 +fi +#读取第一个参数 为配置文件目录名称 +read_dir $1 + diff --git a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/stop.sh b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/stop.sh new file mode 100644 index 0000000..a3b21d6 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/stop.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#flink任务停止脚本 +source /etc/profile + + +function read_dir(){ + for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令 + do + if [ -d $1"/"$file ] #注意此处之间一定要加上空格，否则会报错 + then + read_dir $1"/"$file + else + jobid=`flink list | grep "$file" | awk '{print $4}'` + flink cancel $jobid + fi +done +} +#读取第一个参数 为配置文件目录名 +read_dir $1 diff --git a/TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties b/TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties new file mode 100644 index 0000000..57b5c9c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties @@ -0,0 +1,5 @@ +# dos任务新增以下配置 + + +#baseline ttl，单位：天 +hbase.baseline.ttl=30 diff --git a/TSG发布版本更新记录/TSG-21.12/flink/topN/kafka-flinksql-top.sql b/TSG发布版本更新记录/TSG-21.12/flink/topN/kafka-flinksql-top.sql new file mode 100644 index 0000000..877ac27 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/topN/kafka-flinksql-top.sql @@ -0,0 +1,415 @@ +--通联： +CREATE TABLE session_record_completed_log( +common_schema_type VARCHAR, +common_recv_time BIGINT, +common_client_ip VARCHAR, +common_server_ip VARCHAR, +http_host VARCHAR, +http_domain VARCHAR, +common_l4_protocol VARCHAR, +common_internal_ip VARCHAR, +common_external_ip VARCHAR, +common_subscriber_id VARCHAR, +common_app_label VARCHAR, +common_sessions BIGINT, +common_c2s_pkt_num BIGINT, +common_s2c_pkt_num BIGINT, +common_c2s_byte_num BIGINT, +common_s2c_byte_num BIGINT, +common_processing_time BIGINT, +stat_time as TO_TIMESTAMP(FROM_UNIXTIME(common_recv_time)), +WATERMARK FOR stat_time AS stat_time - INTERVAL '1' MINUTE) +WITH( +'connector' = 'kafka', +'properties.group.id' = 'kafka-indexing-service', +'topic' = 'SESSION-RECORD-COMPLETED', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.security.protocol'='SASL_PLAINTEXT', +'properties.sasl.mechanism'='PLAIN', +'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'scan.startup.mode' = 'latest-offset', +'sink.parallelism'='1', +--'sink.parallelism'='60', +'format' = 'json' +); + +--client： +CREATE TABLE top_client_ip_log( +source VARCHAR, +session_num BIGINT, +c2s_pkt_num BIGINT, +s2c_pkt_num BIGINT, +c2s_byte_num BIGINT, +s2c_byte_num BIGINT, +order_by VARCHAR, +stat_time BIGINT, +PRIMARY KEY (stat_time) NOT ENFORCED +)WITH( +'connector' = 'upsert-kafka', +'topic' = 'TOP-CLIENT-IP', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.security.protocol'='SASL_PLAINTEXT', +'properties.sasl.mechanism'='PLAIN', +'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +--'sink.parallelism'='1', +'key.format' = 'json', +'value.format' = 'json' +); + +CREATE VIEW top_client_ip_view as +SELECT common_client_ip as source,sum(common_sessions) as session_num,sum(common_c2s_pkt_num) as c2s_pkt_num,sum(common_s2c_pkt_num) as s2c_pkt_num,sum(common_c2s_byte_num) as c2s_byte_num,sum(common_s2c_byte_num) as s2c_byte_num,UNIX_TIMESTAMP(CAST(TUMBLE_END(stat_time,INTERVAL '5' MINUTE) as VARCHAR)) as stat_time +FROM session_record_completed_log +where common_l4_protocol = 'IPv6_TCP' or common_l4_protocol = 'IPv4_TCP' +group by common_client_ip,TUMBLE(stat_time,INTERVAL '5' MINUTE); + +INSERT INTO top_client_ip_log +(SELECT `source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'sessions' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY session_num DESC) as rownum +FROM +top_client_ip_view) +WHERE rownum <= 1000) +union all +(SELECT `source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'packets' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_pkt_num+s2c_pkt_num DESC) as rownum +FROM +top_client_ip_view) +WHERE rownum <= 1000) +union all +(SELECT `source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'bytes' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_byte_num+s2c_byte_num DESC) as rownum +FROM +top_client_ip_view) +WHERE rownum <= 1000); + + + + +--server: +CREATE TABLE top_server_ip_log( +destination VARCHAR, +session_num BIGINT, +c2s_pkt_num BIGINT, +s2c_pkt_num BIGINT, +c2s_byte_num BIGINT, +s2c_byte_num BIGINT, +order_by VARCHAR, +stat_time BIGINT, +PRIMARY KEY (stat_time) NOT ENFORCED +)WITH( +'connector' = 'upsert-kafka', +'topic' = 'TOP-SERVER-IP', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.security.protocol'='SASL_PLAINTEXT', +'properties.sasl.mechanism'='PLAIN', +'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +--'sink.parallelism'='1', +'key.format' = 'json', +'value.format' = 'json' +); + +CREATE VIEW top_server_ip_view as +SELECT common_server_ip as `destination`,sum(common_sessions) as session_num,sum(common_c2s_pkt_num) as c2s_pkt_num,sum(common_s2c_pkt_num) as s2c_pkt_num,sum(common_c2s_byte_num) as c2s_byte_num,sum(common_s2c_byte_num) as s2c_byte_num,UNIX_TIMESTAMP(CAST(TUMBLE_END(stat_time,INTERVAL '5' MINUTE) as VARCHAR)) as stat_time +FROM session_record_completed_log +where common_l4_protocol = 'IPv6_TCP' or common_l4_protocol = 'IPv4_TCP' +group by common_server_ip,TUMBLE(stat_time,INTERVAL '5' MINUTE); + +INSERT INTO top_server_ip_log +(SELECT `destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'sessions' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY session_num DESC) as rownum +FROM +top_server_ip_view) +WHERE rownum <= 1000) +union all +(SELECT `destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'packets' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_pkt_num+s2c_pkt_num DESC) as rownum +FROM +top_server_ip_view) +WHERE rownum <= 1000) +union all +(SELECT destination, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +destination, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'bytes' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_byte_num+s2c_byte_num DESC) as rownum +FROM +top_server_ip_view) +WHERE rownum <= 1000); + + +--internal +CREATE TABLE top_internal_ip_log ( + source VARCHAR, + session_num BIGINT, + c2s_pkt_num BIGINT, + s2c_pkt_num BIGINT, + c2s_byte_num BIGINT, + s2c_byte_num BIGINT, + order_by VARCHAR, + stat_time BIGINT, + PRIMARY KEY (stat_time) NOT ENFORCED +) WITH ( +'connector' = 'upsert-kafka', +'topic' = 'TOP-INTERNAL-HOST', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.security.protocol'='SASL_PLAINTEXT', +'properties.sasl.mechanism'='PLAIN', +'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +--'sink.parallelism'='1', +'key.format' = 'json', +'value.format' = 'json' +); + +CREATE VIEW top_common_internal_ip_view as +SELECT common_internal_ip as `source`,sum(common_sessions) as session_num,sum(common_c2s_pkt_num) as c2s_pkt_num,sum(common_s2c_pkt_num) as s2c_pkt_num,sum(common_c2s_byte_num) as c2s_byte_num,sum(common_s2c_byte_num) as s2c_byte_num,UNIX_TIMESTAMP(CAST(TUMBLE_END(stat_time,INTERVAL '5' MINUTE) as VARCHAR)) as stat_time +FROM session_record_completed_log +where common_internal_ip<>'' +group by common_internal_ip,TUMBLE(stat_time,INTERVAL '5' MINUTE); + + +INSERT INTO top_internal_ip_log +(SELECT `source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'sessions' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY session_num DESC) as rownum +FROM +top_common_internal_ip_view) +WHERE rownum <= 1000) +union all +(SELECT `source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'packets' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_pkt_num+s2c_pkt_num DESC) as rownum +FROM +top_common_internal_ip_view) +WHERE rownum <= 1000) +union all +(SELECT `source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`source`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'bytes' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_byte_num+s2c_byte_num DESC) as rownum +FROM +top_common_internal_ip_view) +WHERE rownum <= 1000); + + +--external: +CREATE TABLE top_external_ip_log ( + destination VARCHAR, + session_num BIGINT, + c2s_pkt_num BIGINT, + s2c_pkt_num BIGINT, + c2s_byte_num BIGINT, + s2c_byte_num BIGINT, + order_by VARCHAR, + stat_time BIGINT, + PRIMARY KEY (stat_time) NOT ENFORCED +) WITH ( +'connector' = 'upsert-kafka', +'topic' = 'TOP-EXTERNAL-HOST', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.security.protocol'='SASL_PLAINTEXT', +'properties.sasl.mechanism'='PLAIN', +'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +--'sink.parallelism'='1', +'key.format' = 'json', +'value.format' = 'json' +); + + +CREATE VIEW top_common_external_ip_view as +SELECT common_external_ip as `destination`,sum(common_sessions) as session_num,sum(common_c2s_pkt_num) as c2s_pkt_num,sum(common_s2c_pkt_num) as s2c_pkt_num,sum(common_c2s_byte_num) as c2s_byte_num,sum(common_s2c_byte_num) as s2c_byte_num,UNIX_TIMESTAMP(CAST(TUMBLE_END(stat_time,INTERVAL '5' MINUTE) as VARCHAR)) as stat_time +FROM session_record_completed_log +where common_external_ip<>'' +group by common_external_ip,TUMBLE(stat_time,INTERVAL '5' MINUTE); + + +INSERT INTO top_external_ip_log +(SELECT `destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'sessions' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY session_num DESC) as rownum +FROM +top_common_external_ip_view) +WHERE rownum <= 1000) +union all +(SELECT `destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'packets' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_pkt_num+s2c_pkt_num DESC) as rownum +FROM +top_common_external_ip_view) +WHERE rownum <= 1000) +union all +(SELECT `destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`destination`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'bytes' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_byte_num+s2c_byte_num DESC) as rownum +FROM +top_common_external_ip_view) +WHERE rownum <= 1000); + + +--website_domain +CREATE TABLE top_website_domain_log ( + domain VARCHAR, + session_num BIGINT, + c2s_pkt_num BIGINT, + s2c_pkt_num BIGINT, + c2s_byte_num BIGINT, + s2c_byte_num BIGINT, + order_by VARCHAR, + stat_time BIGINT, + PRIMARY KEY (stat_time) NOT ENFORCED +) WITH ( +'connector' = 'upsert-kafka', +'topic' = 'TOP-WEBSITE-DOMAIN', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.security.protocol'='SASL_PLAINTEXT', +'properties.sasl.mechanism'='PLAIN', +'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +--'sink.parallelism'='1', +'key.format' = 'json', +'value.format' = 'json' +); + +CREATE VIEW top_website_domain_view as +SELECT http_domain as `domain`,sum(common_sessions) as session_num,sum(common_c2s_pkt_num) as c2s_pkt_num,sum(common_s2c_pkt_num) as s2c_pkt_num,sum(common_c2s_byte_num) as c2s_byte_num,sum(common_s2c_byte_num) as s2c_byte_num,UNIX_TIMESTAMP(CAST(TUMBLE_END(stat_time,INTERVAL '5' MINUTE) as VARCHAR)) as stat_time +FROM session_record_completed_log +where http_domain<>'' +group by http_domain,TUMBLE(stat_time,INTERVAL '5' MINUTE); + + +INSERT INTO top_website_domain_log +(SELECT `domain`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`domain`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'sessions' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY session_num DESC) as rownum +FROM +top_website_domain_view) +WHERE rownum <= 1000) +union all +(SELECT `domain`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`domain`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'packets' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_pkt_num+s2c_pkt_num DESC) as rownum +FROM +top_website_domain_view) +WHERE rownum <= 1000) +union all +(SELECT `domain`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`domain`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'bytes' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_byte_num+s2c_byte_num DESC) as rownum +FROM +top_website_domain_view) +WHERE rownum <= 1000); + + +--user: +CREATE TABLE top_user_log ( + subscriber_id VARCHAR, + session_num BIGINT, + c2s_pkt_num BIGINT, + s2c_pkt_num BIGINT, + c2s_byte_num BIGINT, + s2c_byte_num BIGINT, + order_by VARCHAR, + stat_time BIGINT, + PRIMARY KEY (stat_time) NOT ENFORCED +) WITH ( +'connector' = 'upsert-kafka', +'topic' = 'TOP-USER', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.security.protocol'='SASL_PLAINTEXT', +'properties.sasl.mechanism'='PLAIN', +'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +--'sink.parallelism'='1', +'key.format' = 'json', +'value.format' = 'json' +); + +CREATE VIEW top_user_log_view as +SELECT common_subscriber_id as `subscriber_id`,sum(common_sessions) as session_num,sum(common_c2s_pkt_num) as c2s_pkt_num,sum(common_s2c_pkt_num) as s2c_pkt_num,sum(common_c2s_byte_num) as c2s_byte_num,sum(common_s2c_byte_num) as s2c_byte_num,UNIX_TIMESTAMP(CAST(TUMBLE_END(stat_time,INTERVAL '5' MINUTE) as VARCHAR)) as stat_time +FROM session_record_completed_log +where common_subscriber_id <>'' +group by common_subscriber_id,TUMBLE(stat_time,INTERVAL '5' MINUTE); + +INSERT INTO top_user_log +(SELECT `subscriber_id`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`subscriber_id`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'sessions' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY session_num DESC) as rownum +FROM +top_user_log_view) +WHERE rownum <= 1000) +union all +(SELECT `subscriber_id`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`subscriber_id`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'packets' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_pkt_num+s2c_pkt_num DESC) as rownum +FROM +top_user_log_view) +WHERE rownum <= 1000) +union all +(SELECT `subscriber_id`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,order_by,stat_time FROM +(SELECT +`subscriber_id`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,'bytes' as order_by,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ORDER BY c2s_byte_num+s2c_byte_num DESC) as rownum +FROM +top_user_log_view) +WHERE rownum <= 1000); + + + + + +--app +CREATE TABLE top_app_log ( + app_name VARCHAR, + session_num BIGINT, + c2s_pkt_num BIGINT, + s2c_pkt_num BIGINT, + c2s_byte_num BIGINT, + s2c_byte_num BIGINT, + stat_time BIGINT, + PRIMARY KEY (stat_time) NOT ENFORCED +) WITH ( +'connector' = 'upsert-kafka', +'topic' = 'TRAFFIC-APP-STAT', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +--'properties.bootstrap.servers' = '10.111.136.193:9092,10.111.136.194:9092,10.111.136.195:9092,10.111.136.196:9092,10.111.136.197:9092,10.111.136.198:9092,10.111.136.199:9092,10.111.136.200:9092,10.111.136.201:9092,10.111.136.203:9092,10.111.136.204:9092,10.111.136.205:9092,10.111.136.206:9092,10.111.136.207:9092,10.111.136.202:9092', +'properties.security.protocol'='SASL_PLAINTEXT', +'properties.sasl.mechanism'='PLAIN', +'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'key.format' = 'json', +'value.format' = 'json' +); + +CREATE VIEW top_app_log_view as +SELECT common_app_label as `app_name`,sum(common_sessions) as session_num,sum(common_c2s_pkt_num) as c2s_pkt_num,sum(common_s2c_pkt_num) as s2c_pkt_num,sum(common_c2s_byte_num) as c2s_byte_num,sum(common_s2c_byte_num) as s2c_byte_num,UNIX_TIMESTAMP(CAST(TUMBLE_END(stat_time,INTERVAL '5' MINUTE) as VARCHAR)) as stat_time +FROM session_record_completed_log +where common_app_label<>'' +group by common_app_label,TUMBLE(stat_time,INTERVAL '5' MINUTE); + +INSERT INTO top_app_log +(SELECT `app_name`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,stat_time FROM +(SELECT +`app_name`, session_num, c2s_pkt_num,s2c_pkt_num,c2s_byte_num,s2c_byte_num,stat_time, +ROW_NUMBER() OVER (PARTITION BY stat_time ) +FROM +top_app_log_view)); + + + + + + diff --git a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties new file mode 100644 index 0000000..9e96c4e --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties @@ -0,0 +1,38 @@ +#kafka的地址信息 +source.kafka.broker=192.168.44.11:9094 +source.kafka.group.id =vpn-1206-1 +source.kafka.topic=SESSION-RECORD-COMPLETED +source.kafka.parallelism=1 +max.poll.records=3000 +session.timeout.ms=60000 +max.partition.fetch.bytes=31457280 +#hbase的zk地址 +zk.host=192.168.44.11:2181 +#写入hbase并行度 +sink.hbase.parallelism=1 +#写入hbase列簇 +sink.hbase.fm=common +#写入hbase表名 +sink.hbase.table=tsg_galaxy:recommendation_app_cip +#任务并行度 +task.parallelism=1 +#窗口延迟等待时间单位秒 +watermark.time=1 +#top结果限制 +top.limit=10000 +#滑动窗口总时间单位分钟 +slidingwindow.time.minute=30 +#每个滑块时间单位分钟 +slidingwindowslot.time.minute=1 +#kafka是否开启安全验证 0不开启 1SSL 2 SASL +kafka.security=2 +#kafka SASL验证用户名 +kafka.user=admin +#kafka SASL及SSL验证密码 +kafka.pin=galaxy2019 +#1SSL需要 +tools.library=D:\\K18-Phase2\\tsgSpace\\dat\\tsg\\ +#是否接受全量app 无过滤条件false 白名单过滤true +has.filter=false +#只计算filter命中的common_app_label逗号分隔 baidu.com,qq 可不填写 +app.white.list= diff --git a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/config/RECOMMENDATION-APP-CIP b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/config/RECOMMENDATION-APP-CIP new file mode 100644 index 0000000..9e96c4e --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/config/RECOMMENDATION-APP-CIP @@ -0,0 +1,38 @@ +#kafka的地址信息 +source.kafka.broker=192.168.44.11:9094 +source.kafka.group.id =vpn-1206-1 +source.kafka.topic=SESSION-RECORD-COMPLETED +source.kafka.parallelism=1 +max.poll.records=3000 +session.timeout.ms=60000 +max.partition.fetch.bytes=31457280 +#hbase的zk地址 +zk.host=192.168.44.11:2181 +#写入hbase并行度 +sink.hbase.parallelism=1 +#写入hbase列簇 +sink.hbase.fm=common +#写入hbase表名 +sink.hbase.table=tsg_galaxy:recommendation_app_cip +#任务并行度 +task.parallelism=1 +#窗口延迟等待时间单位秒 +watermark.time=1 +#top结果限制 +top.limit=10000 +#滑动窗口总时间单位分钟 +slidingwindow.time.minute=30 +#每个滑块时间单位分钟 +slidingwindowslot.time.minute=1 +#kafka是否开启安全验证 0不开启 1SSL 2 SASL +kafka.security=2 +#kafka SASL验证用户名 +kafka.user=admin +#kafka SASL及SSL验证密码 +kafka.pin=galaxy2019 +#1SSL需要 +tools.library=D:\\K18-Phase2\\tsgSpace\\dat\\tsg\\ +#是否接受全量app 无过滤条件false 白名单过滤true +has.filter=false +#只计算filter命中的common_app_label逗号分隔 baidu.com,qq 可不填写 +app.white.list= diff --git a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/start.sh b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/start.sh new file mode 100644 index 0000000..14a8aa4 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/start.sh @@ -0,0 +1,33 @@ +#! /bin/bash +#启动storm任务脚本 +source /etc/profile +#任务jar所在目录 +BASE_DIR=`pwd` +#jar name +JAR_NAME='flink-vpn-recommend-21-12-06.jar' + +#cd $BASE_DIR +jar -xvf $BASE_DIR/$JAR_NAME common.properties +function read_dir(){ + for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令 + do + if [ -d $1"/"$file ] #注意此处之间一定要加上空格，否则会报错 + then + read_dir $1"/"$file + else + num=`flink list | grep "$file" | wc -l` + if [ $num -eq "0" ];then + cat $1$file > $BASE_DIR/common.properties + jar -uvf $BASE_DIR/$JAR_NAME common.properties + flink run -d -p 1 $JAR_NAME + fi + fi +done +} +if [ $# != 1 ];then + echo "usage: ./startall.sh [Configuration path]" + exit 1 +fi +#读取第一个参数 为配置文件目录名称 +read_dir $1 + diff --git a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/stop.sh b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/stop.sh new file mode 100644 index 0000000..a3b21d6 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/stop.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#flink任务停止脚本 +source /etc/profile + + +function read_dir(){ + for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令 + do + if [ -d $1"/"$file ] #注意此处之间一定要加上空格，否则会报错 + then + read_dir $1"/"$file + else + jobid=`flink list | grep "$file" | awk '{print $4}'` + flink cancel $jobid + fi +done +} +#读取第一个参数 为配置文件目录名 +read_dir $1 diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/dos_event.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/dos_event.json new file mode 100644 index 0000000..c17a729 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/dos_event.json @@ -0,0 +1,353 @@ +{ + "type": "record", + "name": "dos_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "log_id", + "partition_key": "start_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + } + }, + "default_columns": [ + "log_id", + "attack_type", + "source_ip_list", + "destination_ip", + "severity", + "start_time", + "end_time", + "packet_rate", + "bit_rate", + "session_rate" + ], + "internal_columns": [ + "common_recv_time", + "common_log_id", + "common_processing_time" + ] + }, + "fields": [ + { + "name": "start_time", + "label": "Start Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "attack_type", + "label": "Attack Type", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "TCP SYN Flood", + "value": "TCP SYN Flood" + }, + { + "code": "UDP Flood", + "value": "UDP Flood" + }, + { + "code": "ICMP Flood", + "value": "ICMP Flood" + }, + { + "code": "DNS Flood", + "value": "DNS Flood" + }, + { + "code": "DNS Amplification", + "value": "DNS Amplification" + } + ] + }, + "type": "string" + }, + { + "name": "severity", + "label": "Severity", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "Critical", + "value": "Critical" + }, + { + "code": "Severe", + "value": "Severe" + }, + { + "code": "Major", + "value": "Major" + }, + { + "code": "Warning", + "value": "Warning" + }, + { + "code": "Minor", + "value": "Minor" + } + ] + }, + "type": "string" + }, + { + "name": "conditions", + "label": "Conditions", + "type": "string" + }, + { + "name": "destination_ip", + "label": "Destination IP", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "destination_country", + "label": "Destination Country", + "type": "string" + }, + { + "name": "source_ip_list", + "label": "Source IPs", + "type": "string" + }, + { + "name": "source_country_list", + "label": "Source Countries", + "type": "string" + }, + { + "name": "session_rate", + "label": "Sessions/s", + "doc": { + "constraints": { + "type": "sessions/sec" + } + }, + "type": "long" + }, + { + "name": "packet_rate", + "label": "Packets/s", + "doc": { + "constraints": { + "type": "packets/sec" + } + }, + "type": "long" + }, + { + "name": "bit_rate", + "label": "Bits/s", + "doc": { + "constraints": { + "type": "bits/sec" + } + }, + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/engine-queries-template.sql b/TSG发布版本更新记录/TSG-21.12/qgw/schema/engine-queries-template.sql new file mode 100644 index 0000000..6b6215b --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/engine-queries-template.sql @@ -0,0 +1,116 @@ +--Q01.CK DateTime +select toDateTime(common_recv_time) as common_recv_time from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 20 +--Q02.Standard DateTime +select FROM_UNIXTIME(common_recv_time) as common_recv_time from session_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) limit 20 +--Q03.count(1) +select count(1) from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) +--Q04.count(*) +select count(*) from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) +--Q05.UDF APPROX_COUNT_DISTINCT_DS_HLL +SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end and policy_id=0 group by policy_id +--Q06.UDF TIME_FLOOR_WITH_FILL +select TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','previous') as stat_time from session_record where common_recv_time > @start and common_recv_time < @end group by stat_time +--Q07.UDF GEO IP +select IP_TO_GEO(common_client_ip) as geo,IP_TO_CITY(common_server_ip) as city,IP_TO_COUNTRY(common_server_ip) as country from session_record limit 10 +--Q08.Special characters +select * from session_record where (common_protocol_label ='/$' or common_client_ip like'%') limit 10 +--Q09.Federation Query +select * from (select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','zero')) as stat_time from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by stat_time order by stat_time asc) +--Q10.Catalog Database +select * from tsg_galaxy_v3.session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 20 +--Q11.Session Record Logs +select * from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) AND @common_filter order by common_recv_time desc limit 20 +--Q12.Live Session Record Logs +select * from interim_session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) AND @common_filter order by common_recv_time desc limit 20 +--Q13.Transaction Record Logs +select * from transaction_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) order by common_recv_time desc limit 20 +--Q14.Security Event Logs +select * from security_event where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) AND @common_filter order by common_recv_time desc limit 0,20 +--Q15.Proxy Event Logs +select * from proxy_event where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q16.Radius Record Logs +select * from radius_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q17.GTPC Record Logs +select * from gtpc_record where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q18.Security Event Logs with fields +select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_log_id,common_policy_id,common_subscriber_id,common_client_ip,common_client_port,common_l4_protocol,common_address_type,common_server_ip,common_server_port,common_action,common_direction,common_sled_ip,common_client_location,common_client_asn,common_server_location,common_server_asn,common_c2s_pkt_num,common_s2c_pkt_num,common_c2s_byte_num,common_s2c_byte_num,common_schema_type,common_sub_action,common_device_id, FROM_UNIXTIME(common_start_time) as common_start_time, FROM_UNIXTIME(common_end_time) as common_end_time,common_establish_latency_ms,common_con_duration_ms,common_stream_dir,common_stream_trace_id,http_url,http_host,http_domain,http_request_body,http_response_body,http_cookie,http_referer,http_user_agent,http_content_length,http_content_type,http_set_cookie,http_version,http_response_latency_ms,http_action_file_size,http_session_duration_ms,mail_protocol_type,mail_account,mail_from_cmd,mail_to_cmd,mail_from,mail_to,mail_cc,mail_bcc,mail_subject,mail_attachment_name,mail_eml_file,dns_message_id,dns_qr,dns_opcode,dns_aa,dns_tc,dns_rd,dns_ra,dns_rcode,dns_qdcount,dns_ancount,dns_nscount,dns_arcount,dns_qname,dns_qtype,dns_qclass,dns_cname,dns_sub,dns_rr,ssl_sni,ssl_san,ssl_cn,ssl_pinningst,ssl_intercept_state,ssl_server_side_latency,ssl_client_side_latency,ssl_server_side_version,ssl_client_side_version,ssl_cert_verify,ssl_error,quic_version,quic_sni,quic_user_agent,ftp_account,ftp_url,ftp_content from security_event where common_recv_time >= @start and common_recv_time < @end order by common_recv_time desc limit 10000 +--Q19.Radius ON/OFF Logs For Frame IP +select framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >=toDateTime(@start) and event_timestamp = @start and event_timestamp < @end group by account +--Q21.Radius ON/OFF Logs total Account number +select count(distinct(framed_ip)) as active_ip_num , sum(acct_session_time) as online_duration from (select any(framed_ip) as framed_ip ,max(acct_session_time) as acct_session_time from radius_onff_log where account='000jS' and event_timestamp >= @start and event_timestamp < @end group by acct_session_id) +--Q22.Radius ON/OFF Logs Account Access Detail +select max(if(acct_status_type=1,event_timestamp,0)) as start_time,max(if(acct_status_type=2,event_timestamp,0)) as end_time, any(framed_ip) as ip,max(acct_session_time) as online_duration from radius_onff_log where event_timestamp >= @start and event_timestamp < @end group by acct_session_id order by start_time desc limit 200 +--Q23.Report for Client IP +select common_client_ip, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@end)) group by common_client_ip order by sessions desc limit 0,100 +--Q24.Report for Server IP +select common_server_ip, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by common_server_ip order by sessions desc limit 0,100 +--Q25.Report for SSL SNI +select ssl_sni, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by ssl_sni order by sessions desc limit 0,100 +--Q26.Report for SSL APP +select common_app_label as applicaiton, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by applicaiton order by sessions desc limit 0,100 +--Q27.Report for Domains +select http_domain AS domain,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(domain) GROUP BY domain ORDER BY bytes DESC LIMIT 100 +--Q28.Report for Domains with unique Client IP +select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_domain, uniq (common_client_ip) as nums from session_record where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_domain in (select http_domain from session_record where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_domain) group by http_domain order by SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_domain order by stat_time asc limit 500 +--Q29. Report for HTTP Host +SELECT http_host as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) GROUP BY host ORDER BY bytes DESC limit 100 union all SELECT 'totals' as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes, SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes, SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) +--Q30.Report for HTTP/HTTPS URLS with Sessions +SELECT http_url AS url,count(*) AS sessions FROM proxy_event WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) GROUP BY url ORDER BY sessions DESC LIMIT 100 +--Q31.Report for HTTP/HTTPS URLS with UNIQUE Client IP +select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_url, count(distinct(common_client_ip)) as nums from proxy_event where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_url IN (select http_url from proxy_event where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) group by http_url order by count(*) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_url order by stat_time asc limit 500 +--Q32.Report for Subscriber ID with Sessions +select common_subscriber_id as user, count(*) as sessions from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) group by common_subscriber_id order by sessions desc limit 0,100 +--Q33.Report for Subscriber ID with Bandwidth +SELECT common_subscriber_id as user,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM session_record WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) GROUP BY user ORDER BY bytes DESC LIMIT 100 +--Q34.Report Unique Endpoints +select uniq(common_client_ip) as "Client IP",uniq(common_server_ip) as "Server IP",uniq(common_internal_ip) as "Internal IP",uniq(common_external_ip) as "External IP",uniq(http_domain) as "Domain",uniq(ssl_sni) as "SNI" from session_record where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) +--Q35.TopN Optimizer +SELECT http_url AS url, SUM(common_sessions) AS sessions FROM session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_url) GROUP BY http_url ORDER BY sessions DESC limit 10 +--Q36.All Security Event Hits Trend by 5min B +select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, sum(hits) as hits from security_event_hits_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') limit 10000 +--Q37.Security Event Hit Time（first and last time) B +select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from security_event_hits_log where policy_id in (0) group by policy_id +--Q38.All Proxy Event Hits Trend by 5min B +select FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end group by FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) limit 10000 +--Q39.Proxy Event Hit Time（first and last time) B +select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from proxy_event_hits_log where policy_id in (0) group by policy_id +--Q40.Traffic Composition Protocol Tree Trend +(SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes from traffic_protocol_stat_log where __time >= TIMESTAMP @start AND __time < TIMESTAMP @end and protocol_id = 'ETHERNET' group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id order by stat_time asc) union all (SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes from traffic_protocol_stat_log where __time >= TIMESTAMP @start AND __time < TIMESTAMP @end and protocol_id like CONCAT('ETHERNET','.%') and LENGTH(protocol_id) = LENGTH(REPLACE(protocol_id,'.','')) + 1 + 0 group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id order by stat_time asc) +--Q41.Traffic Metrics Security Action Hits Trend +select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) as statisticTime, sum(default_in_bytes + default_out_bytes) as default_bytes, sum(default_in_packets + default_out_packets) as default_packets, sum(default_conn_num) as default_sessions, sum(allow_in_bytes + allow_out_bytes) as allow_bytes, sum(allow_in_packets + allow_out_packets) as allow_packets, sum(allow_conn_num) as allow_sessions, sum(deny_in_bytes + deny_out_bytes) as deny_bytes, sum(deny_in_packets + deny_out_packets) as deny_packets, sum(deny_conn_num) as deny_sessions, sum(monitor_in_bytes + monitor_out_bytes) as monitor_bytes, sum(monitor_in_packets + monitor_out_packets) as monitor_packets, sum(monitor_conn_num) as monitor_sessions, sum(intercept_in_bytes + intercept_out_bytes) as intercept_bytes, sum(intercept_in_packets + intercept_out_packets) as intercept_packets, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >= @start and __time < @end group by FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) limit 100000 +--Q42.Traffic Metrics Proxy Action Hits Trend +SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) AS statisticTime,SUM(intcp_allow_num) AS intercept_allow_conn_num,SUM(intcp_mon_num) AS intercept_monitor_conn_num,SUM(intcp_deny_num) AS intercept_deny_conn_num,SUM(intcp_rdirt_num) AS intercept_redirect_conn_num,SUM(intcp_repl_num) AS intercept_replace_conn_num,SUM(intcp_hijk_num) AS intercept_hijack_conn_num,SUM(intcp_ins_num) AS intercept_insert_conn_num FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'PT1800S', 'zero')) LIMIT 100000 +--Q43.Traffic Statistics(Metrics02) +select FROM_UNIXTIME(stat_time) as max_active_date_by_sessions, total_live_sessions as max_live_sessions from ( select stat_time, sum(live_sessions) as total_live_sessions from ( select TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'P1D') as stat_time, device_id, avg(established_conn_num) as live_sessions from traffic_metrics_log where __time >= @start and __time<@end group by TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'P1D'), device_id) group by stat_time order by total_live_sessions desc limit 1 ) +--Q44.Traffic Summary(Bandwidth Trend) +select * from ( select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time,'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'), 'traffic_in_bytes' union all select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time,'traffic_out_bytes' as type,sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'),'traffic_out_bytes' ) order by stat_time asc limit 100000 +--Q45.Traffic Summary(Sessions Trend) +select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time, 'total_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'), 'total_conn_num' order by stat_time asc limit 10000 +--Q46.Domain Baidu.com Metrics +select FROM_UNIXTIME(min(common_recv_time)) as "First Seen" , FROM_UNIXTIME(max(common_recv_time)) as "Last Seen" , median(http_response_latency_ms) as "Server Processing Time Median(ms)", count(1) as Responses,any(common_server_location) as Location from session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND http_domain='baidu.com' +--Q47.TIME_FLOOR_WITH_FILL 01 +select "Device Group" as "Device Group" ,"Data Center" as "Data Center" ,FROM_UNIXTIME("End Time") as "End Time" , sum("counter") as "counter" from (select common_device_group as "Device Group" ,common_data_center as "Data Center" ,TIME_FLOOR_WITH_FILL (common_end_time,'PT1H','zero') as "End Time" ,count(common_log_id) as "counter" from session_record where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by "Device Group","Data Center","End Time") group by "Device Group" ,"Data Center" ,"End Time" order by "End Time" asc limit 5 +--Q48.TIME_FLOOR_WITH_FILL 02 +select FROM_UNIXTIME("End Time") as "End Time" , sum("counter") as "counter" from (select common_device_group as "Device Group" ,common_data_center as "Data Center" ,TIME_FLOOR_WITH_FILL (common_end_time,'PT1H','zero') as "End Time" ,count(common_log_id) as "counter" ,count(http_domain) as "HTTP.Domain" from security_event where ((common_recv_time >= toDateTime('2021-10-19 00:00:00') and common_recv_time < toDateTime('2021-10-20 00:00:00')) ) AND ( ( common_action = 2 ) ) group by "Device Group","Data Center","End Time") group by "End Time" order by "End Time" asc +--Q49.CONVERT_TZ (Druid) 01 +SELECT CONVERT_TZ('2019-09-09 09:09:09','GMT','MET') as test_time from proxy_event_hits_log limit 1 +--Q50.CONVERT_TZ (Druid) 02 +SELECT CONVERT_TZ('2019-09-09 09:09:09','Europe/London','America/New_York') as test_time from proxy_event_hits_log limit 1 +--Q51.CONVERT_TZ (Druid) 03 +SELECT CONVERT_TZ(now(),'GMT','America/New_York') as test_time from proxy_event_hits_log limit 1 +--Q53.CONVERT_TZ (clickhouse) 01 +SELECT CONVERT_TZ('2019-09-09 09:09:09','GMT','MET') as test_time from session_record limit 1 +--Q54.CONVERT_TZ (clickhouse) 02 +SELECT CONVERT_TZ('2019-09-09 09:09:09','Europe/London','America/New_York') as test_time from session_record limit 1 +--Q55.CONVERT_TZ (clickhouse) 03 +SELECT CONVERT_TZ(now(),'GMT','America/New_York') as test_time from session_record limit 1 +--Q57.CONVERT_TZ (hbase) 01 +SELECT CONVERT_TZ('2019-09-09 09:09:09','GMT','MET') as test_time from report_result limit 1 +--Q58.CONVERT_TZ (hbase) 02 +SELECT CONVERT_TZ('2019-09-09 09:09:09','Europe/London','America/New_York') as test_time from report_result limit 1 +--Q59.CONVERT_TZ (hbase) 03 +SELECT CONVERT_TZ(now(),'GMT','America/New_York') as test_time from report_result limit 1 +--Q61.CONVERT_TZ (elasticsearch) +SELECT CONVERT_TZ('2019-09-09 09:09:09','Europe/London','America/New_York')as time from report_result limit 1 \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/gtpc_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/gtpc_record.json new file mode 100644 index 0000000..f2164f7 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/gtpc_record.json @@ -0,0 +1,1191 @@ +{ + "type": "record", + "name": "gtpc_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number", + "gtp_msg_type" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number", + "gtp_end_user_ipv4", + "gtp_end_user_ipv6", + "gtp_uplink_teid", + "gtp_downlink_teid", + "gtp_msg_type" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "GTP-C": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_app_surrogate_id", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number", + "gtp_end_user_ipv4", + "gtp_end_user_ipv6", + "gtp_uplink_teid", + "gtp_downlink_teid", + "gtp_msg_type" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "gtp_version", + "gtp_msg_type", + "gtp_imsi", + "gtp_imei", + "gtp_phone_number", + "common_client_ip", + "common_server_ip" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "gtp_version", + "gtp_msg_type", + "gtp_imsi", + "gtp_imei", + "gtp_phone_number", + "common_client_ip", + "common_server_ip" + ], + "internal_columns": [ + "common_recv_time", + "common_log_id", + "common_processing_time" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_client_asn" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "GTP-C", + "value": "GTP-C" + } + ] + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "gtp_version", + "label": "Version", + "type": "string" + }, + { + "name": "gtp_apn", + "label": "APN", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + }, + { + "name": "gtp_msg_type", + "label": "Message Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "create", + "value": "create" + }, + { + "code": "modify", + "value": "modify" + }, + { + "code": "delete", + "value": "delete" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_end_user_ipv4", + "label": "End User Address V4", + "type": "string" + }, + { + "name": "gtp_end_user_ipv6", + "label": "End User Address V6", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-filter.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-filter.json new file mode 100644 index 0000000..d54cf14 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-filter.json @@ -0,0 +1,15 @@ +{ + "version": "1.0", + "name": "hbase-Raw", + "namespace": "tsg", + "filters": [ + { + "name":"@start", + "value": "'2021-10-19 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-10-20 11:00:00'" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-queries-template.sql b/TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-queries-template.sql new file mode 100644 index 0000000..6ff5571 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-queries-template.sql @@ -0,0 +1,4 @@ +--Q01. 范围查询 +SELECT last_update_time FROM relation_account_framedip WHERE last_update_time>=CAST(TO_TIMESTAMP (@start,'yyyy-MM-dd HH:mm:ss','Asia/Shanghai') AS UNSIGNED_LONG) AND last_update_time", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + } + }, + "schema_type": { + "BASE": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "HTTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "http_url", + "http_host", + "http_domain", + "http_request_line", + "http_response_line", + "http_request_header", + "http_response_header", + "http_request_content", + "http_request_content_length", + "http_request_content_type", + "http_response_content", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_request_body_key", + "http_response_body_key", + "http_proxy_flag", + "http_sequence", + "http_snapshot", + "http_cookie", + "http_referer", + "http_user_agent", + "http_content_length", + "http_content_type", + "http_set_cookie", + "http_version", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "http_url", + "common_server_port", + "common_sub_action" + ] + }, + "MAIL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_content", + "mail_content_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_attachment_content", + "mail_eml_file", + "mail_snapshot" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "mail_from", + "mail_to", + "mail_subject" + ] + }, + "DNS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_client_ip", + "dns_qr", + "dns_qname", + "dns_qtype" + ] + }, + "SSL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_pinningst", + "ssl_intercept_state", + "ssl_server_side_latency", + "ssl_client_side_latency", + "ssl_server_side_version", + "ssl_client_side_version", + "ssl_cert_verify", + "ssl_error", + "ssl_con_latency_ms", + "ssl_ja3_fingerprint", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "ssl_sni", + "common_server_ip", + "common_server_port" + ] + }, + "QUIC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "quic_version", + "quic_sni", + "quic_user_agent" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "quic_sni", + "common_server_ip", + "common_server_port" + ] + }, + "FTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "ftp_account", + "ftp_url", + "ftp_content", + "ftp_link_type" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "ftp_url", + "common_server_ip", + "common_server_port" + ] + }, + "BGP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "bgp_type", + "bgp_as_num", + "bgp_route" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "bgp_type", + "bgp_as_num", + "common_server_ip", + "common_server_port" + ] + }, + "SIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port" + ] + }, + "RTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "APP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "app_extra_info" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_app_id", + "common_app_label", + "app_extra_info", + "common_server_ip", + "common_server_port" + ] + }, + "DoH": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "doh_url", + "doh_host", + "doh_request_line", + "doh_response_line", + "doh_cookie", + "doh_referer", + "doh_user_agent", + "doh_content_length", + "doh_content_type", + "doh_set_cookie", + "doh_version", + "doh_message_id", + "doh_qr", + "doh_opcode", + "doh_aa", + "doh_tc", + "doh_rd", + "doh_ra", + "doh_rcode", + "doh_qdcount", + "doh_ancount", + "doh_nscount", + "doh_arcount", + "doh_qname", + "doh_qtype", + "doh_qclass", + "doh_cname", + "doh_sub", + "doh_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_client_ip", + "doh_url", + "doh_qname", + "common_server_port" + ] + }, + "VoIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "SSH": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "ssh_auth_success" + ] + }, + "RADIUS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_device_group", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "common_mirrored_pkts", + "common_mirrored_bytes", + "radius_packet_type", + "radius_nas_ip", + "radius_framed_ip", + "radius_account", + "radius_session_timeout", + "radius_idle_timeout", + "radius_acct_status_type", + "radius_acct_terminate_cause", + "radius_event_timestamp", + "radius_nas_port", + "radius_service_type", + "radius_framed_protocol", + "radius_callback_number", + "radius_callback_id", + "radius_termination_action", + "radius_called_station_id", + "radius_calling_station_id", + "radius_acct_delay_time", + "radius_acct_session_id", + "radius_acct_multi_session_id", + "radius_acct_input_octets", + "radius_acct_output_octets", + "radius_acct_input_packets", + "radius_acct_output_packets", + "radius_acct_session_time", + "radius_acct_link_count", + "radius_acct_interim_interval", + "radius_acct_authentic" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "radius_nas_ip", + "radius_framed_ip", + "radius_acct_status_type" + ] + } + }, + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ], + "L2TP": [ + { + "name": "l2tp_version", + "label": "Version", + "type": "string" + }, + { + "name": "l2tp_lac2lns_tunnel_id", + "label": "LAC2LNS Tunnel ID", + "type": "int" + }, + { + "name": "l2tp_lns2lac_tunnel_id", + "label": "LNS2LAC Tunnel ID", + "type": "int" + }, + { + "name": "l2tp_lac2lns_session_id", + "label": "LAC2LNS Session ID", + "type": "int" + }, + { + "name": "l2tp_lns2lac_session_id", + "label": "LNS2LAC Session ID", + "type": "int" + }, + { + "name": "l2tp_access_concentrator_ip", + "label": "Access Concentrator IP", + "type": "string" + }, + { + "name": "l2tp_access_concentrator_port", + "label": "Access Concentrator Port", + "type": "int" + }, + { + "name": "l2tp_network_server_ip", + "label": "Network Server IP", + "type": "string" + }, + { + "name": "l2tp_network_server_port", + "label": "Network Server Port", + "type": "int" + } + ], + "PPTP": [ + { + "name": "pptp_uplink_tunnel_id", + "label": "UpLink Tunnel ID", + "type": "int" + }, + { + "name": "pptp_downlink_tunnel_id", + "label": "Down Tunnel ID", + "type": "int" + } + ] + }, + "fields": { + "common_encapsulation": { + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ] + }, + "common_has_dup_traffic": { + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/radius_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/radius_record.json new file mode 100644 index 0000000..58a7f37 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/radius_record.json @@ -0,0 +1,1270 @@ +{ + "type": "record", + "name": "radius_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "radius_nas_ip", + "radius_framed_ip", + "common_subscriber_id" + ], + "metrics": [ + "radius_framed_ip", + "radius_event_timestamp", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num" + ], + "filters": [ + "radius_framed_ip", + "common_subscriber_id", + "radius_packet_type", + "radius_acct_session_id", + "radius_acct_multi_session_id", + "radius_acct_status_type" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "RADIUS": { + "$ref": "public_schema_info.json#/schema_type/RADIUS" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "radius_nas_ip", + "radius_framed_ip", + "radius_acct_status_type" + ], + "internal_columns": [ + "common_recv_time", + "common_log_id", + "common_processing_time" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "48", + "value": "Manipulation" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "FTP", + "value": "FTP" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + }, + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "radius_packet_type", + "label": "Packet Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Access-Request" + }, + { + "code": "2", + "value": "Access-Accept" + }, + { + "code": "3", + "value": "Access-Reject" + }, + { + "code": "4", + "value": "Accounting-Request" + }, + { + "code": "5", + "value": "Accounting-Response" + }, + { + "code": "11", + "value": "Access-Challenge" + } + ] + }, + "type": "int" + }, + { + "name": "radius_account", + "label": "Account", + "doc": { + "format": { + "functions": "get_value", + "appendTo": "common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "radius_nas_ip", + "label": "Nas IP", + "type": "string" + }, + { + "name": "radius_framed_ip", + "label": "Framed IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "radius_session_timeout", + "label": "Session Timeout", + "type": "int" + }, + { + "name": "radius_idle_timeout", + "label": "Idle Timeout", + "type": "int" + }, + { + "name": "radius_acct_status_type", + "label": "ACC Status Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Start" + }, + { + "code": "2", + "value": "Stop" + }, + { + "code": "3", + "value": "Interim-Update" + }, + { + "code": "7", + "value": "Accounting-On" + }, + { + "code": "8", + "value": "Accounting-Off" + } + ] + }, + "type": "int" + }, + { + "name": "radius_acct_terminate_cause", + "label": "Acct Terminate Cause", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "User Request" + }, + { + "code": "2", + "value": "Lost Carrier" + }, + { + "code": "3", + "value": "Lost Service" + }, + { + "code": "4", + "value": "Idle Timeout" + }, + { + "code": "5", + "value": "Session Timeout" + }, + { + "code": "6", + "value": "Admin Reset" + }, + { + "code": "7", + "value": "Admin Reboot" + }, + { + "code": "8", + "value": "Port Error" + }, + { + "code": "9", + "value": "NAS Error" + }, + { + "code": "10", + "value": "NAS Request" + }, + { + "code": "11", + "value": "NAS Reboot" + }, + { + "code": "12", + "value": "Port Unneeded" + }, + { + "code": "13", + "value": "Port Preempted" + }, + { + "code": "14", + "value": "Port Suspended" + }, + { + "code": "15", + "value": "Service Unavailable" + }, + { + "code": "16", + "value": "Callback" + }, + { + "code": "17", + "value": "User Error" + }, + { + "code": "18", + "value": "Host Request" + } + ] + }, + "type": "int" + }, + { + "name": "radius_event_timestamp", + "label": "Event Timestamp", + "type": "int" + }, + { + "name": "radius_service_type", + "label": "Service Type", + "type": "int" + }, + { + "name": "radius_nas_port", + "label": "Nas Port", + "type": "int" + }, + { + "name": "radius_framed_protocol", + "label": "Framed Protocol", + "type": "int" + }, + { + "name": "radius_callback_number", + "label": "Callback Number", + "type": "string" + }, + { + "name": "radius_callback_id", + "label": "Callback ID", + "type": "string" + }, + { + "name": "radius_termination_action", + "label": "Termination Action", + "type": "int" + }, + { + "name": "radius_called_station_id", + "label": "Called Station ID", + "type": "string" + }, + { + "name": "radius_calling_station_id", + "label": "Calling Station ID", + "type": "string" + }, + { + "name": "radius_acct_delay_time", + "label": "Acct Delay Time", + "type": "int" + }, + { + "name": "radius_acct_session_id", + "label": "Acct Session ID", + "type": "string" + }, + { + "name": "radius_acct_multi_session_id", + "label": "Acct Multi Session ID", + "type": "string" + }, + { + "name": "radius_acct_input_octets", + "label": "Acct Input Octets", + "type": "long" + }, + { + "name": "radius_acct_output_octets", + "label": "Acct Output Octets", + "type": "long" + }, + { + "name": "radius_acct_input_packets", + "label": "Acct Input Packets", + "type": "long" + }, + { + "name": "radius_acct_output_packets", + "label": "Acct Output Packets", + "type": "long" + }, + { + "name": "radius_acct_session_time", + "label": "Acct Session Time", + "type": "int" + }, + { + "name": "radius_acct_link_count", + "label": "Acct Link Count", + "type": "int" + }, + { + "name": "radius_acct_interim_interval", + "label": "Acct Interim Interval", + "type": "int" + }, + { + "name": "radius_acct_authentic", + "label": "Acct Authentic", + "type": "int" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/recommendation_app_cip.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/recommendation_app_cip.json new file mode 100644 index 0000000..d475dd3 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/recommendation_app_cip.json @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "recommendation_app_cip", + "namespace": "tsg_galaxy", + "fields": [ + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/security_event.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/security_event.json new file mode 100644 index 0000000..ce7579e --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/security_event.json @@ -0,0 +1,2429 @@ +{ + "type": "record", + "name": "security_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_policy_id", + "common_action", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_client_side_version", + "ssl_server_side_version", + "ssl_cert_issuer", + "ssl_cert_subject", + "mail_account", + "mail_from", + "mail_to", + "quic_sni", + "quic_version" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_mirrored_pkts", + "common_mirrored_bytes", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_client_side_latency", + "ssl_server_side_latency", + "ssl_cert_issuer", + "ssl_cert_subject", + "mail_account", + "mail_from", + "mail_to", + "quic_sni" + ], + "filters": [ + "common_policy_id", + "common_action", + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_mirrored_pkts", + "common_mirrored_bytes", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_direction", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "http_request_content_type", + "http_response_content_type", + "ssl_sni", + "ssl_ja3_hash", + "ssl_pinningst", + "ssl_intercept_state", + "ssl_client_side_version", + "ssl_server_side_version", + "ssl_cert_verify", + "ssl_client_side_latency", + "ssl_server_side_latency", + "ssl_cert_issuer", + "ssl_cert_subject", + "mail_account", + "mail_from", + "mail_to", + "mail_subject", + "quic_sni", + "quic_version" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "$ref": "public_schema_info.json#/schema_type/BASE" + }, + "HTTP": { + "$ref": "public_schema_info.json#/schema_type/HTTP" + }, + "MAIL": { + "$ref": "public_schema_info.json#/schema_type/MAIL" + }, + "DNS": { + "$ref": "public_schema_info.json#/schema_type/DNS" + }, + "SSL": { + "$ref": "public_schema_info.json#/schema_type/SSL" + }, + "QUIC": { + "$ref": "public_schema_info.json#/schema_type/QUIC" + }, + "FTP": { + "$ref": "public_schema_info.json#/schema_type/FTP" + }, + "BGP": { + "$ref": "public_schema_info.json#/schema_type/BGP" + }, + "SIP": { + "$ref": "public_schema_info.json#/schema_type/SIP" + }, + "RTP": { + "$ref": "public_schema_info.json#/schema_type/RTP" + }, + "APP": { + "$ref": "public_schema_info.json#/schema_type/APP" + }, + "SSH": { + "$ref": "public_schema_info.json#/schema_type/SSH" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "internal_columns": [ + "common_recv_time", + "common_log_id", + "common_processing_time" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "doc": { + "format": { + "functions": "set_value", + "param": "1" + } + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "QUIC", + "value": "QUIC" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + }, + { + "code": "APP", + "value": "APP" + }, + { + "code": "SSH", + "value": "SSH" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string" + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "allow_query": "true" + } + }, + { + "name": "http_url", + "label": "HTTP.URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "label": "HTTP.Response Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Header", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Header", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content_length", + "label": "HTTP.Request Content Length", + "type": "string" + }, + { + "name": "http_request_content_type", + "label": "HTTP.Request Content Type", + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_content_length", + "label": "HTTP.Response Content Length", + "type": "string" + }, + { + "name": "http_response_content_type", + "label": "HTTP.Response Content Type", + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "mail_protocol_type", + "label": "Mail.Protocol Type", + "type": "string" + }, + { + "name": "mail_account", + "label": "Mail.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "mail_from_cmd", + "label": "Mail.From CMD", + "type": "string" + }, + { + "name": "mail_to_cmd", + "label": "Mail.To CMD", + "type": "string" + }, + { + "name": "mail_from", + "label": "Mail.From", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_to", + "label": "Mail.To", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_cc", + "label": "Mail.CC", + "type": "string" + }, + { + "name": "mail_bcc", + "label": "Mail.BCC", + "type": "string" + }, + { + "name": "mail_subject", + "label": "Mail.Subject", + "doc": { + "allow_query": "true", + "format": { + "functions": "decode_of_base64", + "param": "$.mail_subject_charset" + } + }, + "type": "string" + }, + { + "name": "mail_subject_charset", + "label": "Mail.Subject Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content", + "label": "Mail.Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content_charset", + "label": "Mail.Content Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_name", + "label": "Mail.Attachment", + "doc": { + "format": { + "functions": "decode_of_base64", + "param": "$.mail_attachment_name_charset" + } + }, + "type": "string" + }, + { + "name": "mail_attachment_name_charset", + "label": "Mail.Attachment Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_content", + "label": "Mail.Attachment Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_eml_file", + "label": "Mail.EML File", + "doc": { + "constraints": { + "type": "file" + }, + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_snapshot", + "label": "Mail.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "dns_message_id", + "label": "DNS.Message ID", + "type": "int" + }, + { + "name": "dns_qr", + "label": "DNS.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_opcode", + "label": "DNS.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_aa", + "label": "DNS.AA", + "type": "int" + }, + { + "name": "dns_tc", + "label": "DNS.TC", + "type": "int" + }, + { + "name": "dns_rd", + "label": "DNS.RD", + "type": "int" + }, + { + "name": "dns_ra", + "label": "DNS.RA", + "type": "int" + }, + { + "name": "dns_rcode", + "label": "DNS.RCODE", + "type": "int" + }, + { + "name": "dns_qdcount", + "label": "DNS.QDCOUNT", + "type": "int" + }, + { + "name": "dns_ancount", + "label": "DNS.ANCOUNT", + "type": "int" + }, + { + "name": "dns_nscount", + "label": "DNS.NSCOUNT", + "type": "int" + }, + { + "name": "dns_arcount", + "label": "DNS.ARCOUNT", + "type": "int" + }, + { + "name": "dns_qname", + "label": "DNS.QNAME", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "dns_qtype", + "label": "DNS.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "dns_qclass", + "label": "DNS.QCLASS", + "type": "int" + }, + { + "name": "dns_cname", + "label": "DNS.CNAME", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "dns_sub", + "label": "DNS.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "dns_rr", + "label": "DNS.RR", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_version", + "label": "SSL.Version", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_sni", + "label": "SSL.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "ssl_san", + "label": "SSL.SAN", + "type": "string" + }, + { + "name": "ssl_cn", + "label": "SSL.CN", + "type": "string" + }, + { + "name": "ssl_pinningst", + "label": "SSL.Pinning", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Not Pinning" + }, + { + "code": "1", + "value": "Pinning" + }, + { + "code": "2", + "value": "Maybe Pinning" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_intercept_state", + "label": "SSL.Intercept State", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Passthrough" + }, + { + "code": "1", + "value": "Intercept" + }, + { + "code": "2", + "value": "Shutdown" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_server_side_latency", + "label": "SSL.Server Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_client_side_latency", + "label": "SSL.Client Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_server_side_version", + "label": "SSL.Server Side Version", + "type": "string" + }, + { + "name": "ssl_client_side_version", + "label": "SSL.Client Side Version", + "type": "string" + }, + { + "name": "ssl_cert_verify", + "label": "SSL.Certificate Verify", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_error", + "label": "SSL.Error", + "type": "string" + }, + { + "name": "ssl_con_latency_ms", + "label": "SSL.Connection Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "ssl_ja3_fingerprint", + "label": "SSL.JA3", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ssl_ja3_hash", + "label": "SSL.JA3 hash", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ssl_cert_issuer", + "label": "SSL.Issuer", + "doc": { + "allow_query": "true", + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "ssl_cert_subject", + "label": "SSL.Subject", + "doc": { + "allow_query": "true", + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "quic_version", + "label": "Quic.Version", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "quic_sni", + "label": "Quic.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "quic_user_agent", + "label": "Quic.User Agent", + "type": "string" + }, + { + "name": "ftp_account", + "label": "FTP.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ftp_url", + "label": "FTP.URL", + "type": "string" + }, + { + "name": "ftp_content", + "label": "FTP.Content", + "type": "string" + }, + { + "name": "ftp_link_type", + "label": "FTP.Link Type", + "type": "string" + }, + { + "name": "bgp_type", + "label": "BGP.Type", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "bgp_as_num", + "label": "BGP.AS Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "bgp_route", + "label": "BGP.Route", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_account", + "label": "VoIP.Calling Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_account", + "label": "VoIP.Called Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_number", + "label": "VoIP.Calling Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_number", + "label": "VoIP.Called Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_url", + "label": "Streaming.Media URL", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_protocol", + "label": "Streaming.Media Protocol", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "app_extra_info", + "label": "APP.Extra Info", + "type": "string" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "ssh_version", + "label": "SSH.Version", + "type": "string" + }, + { + "name": "ssh_auth_success", + "label": "SSH.Authentication Result", + "type": "string" + }, + { + "name": "ssh_client_version", + "label": "SSH.Client Version", + "type": "string" + }, + { + "name": "ssh_server_version", + "label": "SSH.Server Version", + "type": "string" + }, + { + "name": "ssh_cipher_alg", + "label": "SSH.Encryption Algorithm", + "type": "string" + }, + { + "name": "ssh_mac_alg", + "label": "SSH.Signing Algorithm", + "type": "string" + }, + { + "name": "ssh_compression_alg", + "label": "SSH.Compression Algorithm", + "type": "string" + }, + { + "name": "ssh_kex_alg", + "label": "SSH. Key Exchange Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key_alg", + "label": "SSH.Server Host Key Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key", + "label": "SSH.Server Key Fingerprint", + "type": "string" + }, + { + "name": "ssh_hassh", + "label": "SSH.HASSH", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/session_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/session_record.json new file mode 100644 index 0000000..726761c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/session_record.json @@ -0,0 +1,2364 @@ +{ + "type": "record", + "name": "session_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "index_table": "session_record_common_client_ip,session_record_common_server_ip,session_record_http_domain", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "quic_sni", + "quic_version" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "quic_sni" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_cookie", + "http_referer", + "http_user_agent", + "ssl_sni", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "quic_sni", + "quic_version" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "$ref": "public_schema_info.json#/schema_type/BASE" + }, + "HTTP": { + "$ref": "public_schema_info.json#/schema_type/HTTP" + }, + "MAIL": { + "$ref": "public_schema_info.json#/schema_type/MAIL" + }, + "DNS": { + "$ref": "public_schema_info.json#/schema_type/DNS" + }, + "SSL": { + "$ref": "public_schema_info.json#/schema_type/SSL" + }, + "QUIC": { + "$ref": "public_schema_info.json#/schema_type/QUIC" + }, + "FTP": { + "$ref": "public_schema_info.json#/schema_type/FTP" + }, + "BGP": { + "$ref": "public_schema_info.json#/schema_type/BGP" + }, + "APP": { + "$ref": "public_schema_info.json#/schema_type/APP" + }, + "SSH": { + "$ref": "public_schema_info.json#/schema_type/SSH" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "internal_columns": [ + "common_recv_time", + "common_log_id", + "common_processing_time" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "QUIC", + "value": "QUIC" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "APP", + "value": "APP" + }, + { + "code": "SSH", + "value": "SSH" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string" + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "http_url", + "label": "HTTP.URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "label": "HTTP.Response Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content_length", + "label": "HTTP.Request Content Length", + "type": "string" + }, + { + "name": "http_request_content_type", + "label": "HTTP.Request Content Type", + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_content_length", + "label": "HTTP.Response Content Length", + "type": "string" + }, + { + "name": "http_response_content_type", + "label": "HTTP.Response Content Type", + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "mail_protocol_type", + "label": "Mail.Protocol Type", + "type": "string" + }, + { + "name": "mail_account", + "label": "Mail.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "mail_from_cmd", + "label": "Mail.From CMD", + "type": "string" + }, + { + "name": "mail_to_cmd", + "label": "Mail.To CMD", + "type": "string" + }, + { + "name": "mail_from", + "label": "Mail.From", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_to", + "label": "Mail.To", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_cc", + "label": "Mail.CC", + "type": "string" + }, + { + "name": "mail_bcc", + "label": "Mail.BCC", + "type": "string" + }, + { + "name": "mail_subject", + "label": "Mail.Subject", + "doc": { + "allow_query": "true", + "format": { + "functions": "decode_of_base64", + "param": "$.mail_subject_charset" + } + }, + "type": "string" + }, + { + "name": "mail_subject_charset", + "label": "Mail.Subject Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content", + "label": "Mail.Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content_charset", + "label": "Mail.Content Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_name", + "label": "Mail.Attachment", + "doc": { + "format": { + "functions": "decode_of_base64", + "param": "$.mail_attachment_name_charset" + } + }, + "type": "string" + }, + { + "name": "mail_attachment_name_charset", + "label": "Mail.Attachment Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_content", + "label": "Mail.Attachment Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_eml_file", + "label": "Mail.EML File", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "mail_snapshot", + "label": "Mail.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "dns_message_id", + "label": "DNS.Message ID", + "type": "int" + }, + { + "name": "dns_qr", + "label": "DNS.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_opcode", + "label": "DNS.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_aa", + "label": "DNS.AA", + "type": "int" + }, + { + "name": "dns_tc", + "label": "DNS.TC", + "type": "int" + }, + { + "name": "dns_rd", + "label": "DNS.RD", + "type": "int" + }, + { + "name": "dns_ra", + "label": "DNS.RA", + "type": "int" + }, + { + "name": "dns_rcode", + "label": "DNS.RCODE", + "type": "int" + }, + { + "name": "dns_qdcount", + "label": "DNS.QDCOUNT", + "type": "int" + }, + { + "name": "dns_ancount", + "label": "DNS.ANCOUNT", + "type": "int" + }, + { + "name": "dns_nscount", + "label": "DNS.NSCOUNT", + "type": "int" + }, + { + "name": "dns_arcount", + "label": "DNS.ARCOUNT", + "type": "int" + }, + { + "name": "dns_qname", + "label": "DNS.QNAME", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "dns_qtype", + "label": "DNS.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "dns_qclass", + "label": "DNS.QCLASS", + "type": "int" + }, + { + "name": "dns_cname", + "label": "DNS.CNAME", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "dns_sub", + "label": "DNS.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "dns_rr", + "label": "DNS.RR", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_version", + "label": "SSL.Version", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_sni", + "label": "SSL.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "ssl_san", + "label": "SSL.SAN", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_cn", + "label": "SSL.CN", + "type": "string" + }, + { + "name": "ssl_pinningst", + "label": "SSL.Pinning", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Not Pinning" + }, + { + "code": "1", + "value": "Pinning" + }, + { + "code": "2", + "value": "Maybe Pinning" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_intercept_state", + "label": "SSL.Intercept State", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Passthrough" + }, + { + "code": "1", + "value": "Intercept" + }, + { + "code": "2", + "value": "Shutdown" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_server_side_latency", + "label": "SSL.Server Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_client_side_latency", + "label": "SSL.Client Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_server_side_version", + "label": "SSL.Server Side Version", + "type": "string" + }, + { + "name": "ssl_client_side_version", + "label": "SSL.Client Side Version", + "type": "string" + }, + { + "name": "ssl_cert_verify", + "label": "SSL.Certificate Verify", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_error", + "label": "SSL.Error", + "type": "string" + }, + { + "name": "ssl_con_latency_ms", + "label": "SSL.Connection Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "ssl_ja3_fingerprint", + "label": "SSL.JA3", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ssl_ja3_hash", + "label": "SSL.JA3 hash", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ssl_cert_issuer", + "label": "SSL.Issuer", + "doc": { + "allow_query": "true", + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "ssl_cert_subject", + "label": "SSL.Subject", + "doc": { + "allow_query": "true", + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "quic_version", + "label": "QUIC.Version", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "quic_sni", + "label": "QUIC.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "quic_user_agent", + "label": "QUIC.User Agent", + "type": "string" + }, + { + "name": "ftp_account", + "label": "FTP.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ftp_url", + "label": "FTP.URL", + "type": "string" + }, + { + "name": "ftp_content", + "label": "FTP.Content", + "type": "string" + }, + { + "name": "ftp_link_type", + "label": "FTP.Link Type", + "type": "string" + }, + { + "name": "bgp_type", + "label": "BGP.Type", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "bgp_as_num", + "label": "BGP.AS Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "bgp_route", + "label": "BGP.Route", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_account", + "label": "VoIP.Calling Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_account", + "label": "VoIP.Called Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_number", + "label": "VoIP.Calling Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_number", + "label": "VoIP.Called Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_url", + "label": "Streaming.Media URL", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_protocol", + "label": "Streaming.Media Protocol", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "app_extra_info", + "label": "APP.Extra Info", + "type": "string" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "files" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "ssh_version", + "label": "SSH.Version", + "type": "string" + }, + { + "name": "ssh_auth_success", + "label": "SSH.Authentication Result", + "type": "string" + }, + { + "name": "ssh_client_version", + "label": "SSH.Client Version", + "type": "string" + }, + { + "name": "ssh_server_version", + "label": "SSH.Server Version", + "type": "string" + }, + { + "name": "ssh_cipher_alg", + "label": "SSH.Encryption Algorithm", + "type": "string" + }, + { + "name": "ssh_mac_alg", + "label": "SSH.Signing Algorithm", + "type": "string" + }, + { + "name": "ssh_compression_alg", + "label": "SSH.Compression Algorithm", + "type": "string" + }, + { + "name": "ssh_kex_alg", + "label": "SSH. Key Exchange Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key_alg", + "label": "SSH.Server Host Key Algorithm", + "type": "string" + }, + { + "name": "ssh_host_key", + "label": "SSH.Server Key Fingerprint", + "type": "string" + }, + { + "name": "ssh_hassh", + "label": "SSH.HASSH", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/transaction_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/transaction_record.json new file mode 100644 index 0000000..43e0a85 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/transaction_record.json @@ -0,0 +1,1515 @@ +{ + "type": "record", + "name": "transaction_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_stream_trace_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "$ref": "public_schema_info.json#/schema_type/BASE" + }, + "HTTP": { + "$ref": "public_schema_info.json#/schema_type/HTTP" + }, + "MAIL": { + "$ref": "public_schema_info.json#/schema_type/MAIL" + }, + "DNS": { + "$ref": "public_schema_info.json#/schema_type/DNS" + }, + "SSL": { + "$ref": "public_schema_info.json#/schema_type/SSL" + }, + "QUIC": { + "$ref": "public_schema_info.json#/schema_type/QUIC" + }, + "FTP": { + "$ref": "public_schema_info.json#/schema_type/FTP" + }, + "BGP": { + "$ref": "public_schema_info.json#/schema_type/BGP" + }, + "SIP": { + "$ref": "public_schema_info.json#/schema_type/SIP" + }, + "RTP": { + "$ref": "public_schema_info.json#/schema_type/RTP" + }, + "APP": { + "$ref": "public_schema_info.json#/schema_type/APP" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "internal_columns": [ + "common_recv_time", + "common_log_id", + "common_processing_time" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "type": "long", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "label": "Receive Time" + }, + { + "name": "common_log_id", + "type": "long", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "label": "Log ID" + }, + { + "name": "common_policy_id", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Policy ID" + }, + { + "name": "common_subscriber_id", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Subscriber ID" + }, + { + "name": "common_imei", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "IMEI" + }, + { + "name": "common_imsi", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "IMSI" + }, + { + "name": "common_phone_number", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Phone Number" + }, + { + "name": "common_client_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "label": "Client IP" + }, + { + "name": "common_internal_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "label": "Internal IP" + }, + { + "name": "common_client_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Client Port" + }, + { + "name": "common_l4_protocol", + "type": "string", + "label": "L4 Protocol" + }, + { + "name": "common_address_type", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "label": "Address Type" + }, + { + "name": "common_server_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "label": "Server IP" + }, + { + "name": "common_server_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Server Port" + }, + { + "name": "common_external_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "label": "External IP" + }, + { + "name": "common_action", + "type": "int", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "label": "Action" + }, + { + "name": "common_direction", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "label": "Direction" + }, + { + "name": "common_entrance_id", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Entrance ID" + }, + { + "name": "common_sled_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Sled IP" + }, + { + "name": "common_client_location", + "type": "string", + "label": "Client Location" + }, + { + "name": "common_client_asn", + "type": "string", + "label": "Client ASN" + }, + { + "name": "common_server_location", + "type": "string", + "label": "Server Location" + }, + { + "name": "common_server_asn", + "type": "string", + "label": "Server ASN" + }, + { + "name": "common_sessions", + "type": "long", + "label": "Sessions" + }, + { + "name": "common_c2s_pkt_num", + "type": "long", + "label": "Packets Sent" + }, + { + "name": "common_s2c_pkt_num", + "type": "long", + "label": "Packets Received" + }, + { + "name": "common_c2s_byte_num", + "type": "long", + "label": "Bytes Sent" + }, + { + "name": "common_s2c_byte_num", + "type": "long", + "label": "Bytes Received" + }, + { + "name": "common_c2s_pkt_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Packets Sent(Diff)" + }, + { + "name": "common_s2c_pkt_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Packets Received(Diff)" + }, + { + "name": "common_c2s_byte_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Bytes Sent(Diff)" + }, + { + "name": "common_s2c_byte_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Bytes Received(Diff)" + }, + { + "name": "common_service", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Service" + }, + { + "name": "common_schema_type", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SIP", + "value": "SIP" + } + ], + "allow_query": "true" + }, + "label": "Schema Type" + }, + { + "name": "common_user_tags", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "User Tags" + }, + { + "name": "common_sub_action", + "type": "string", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "label": "Sub Action" + }, + { + "name": "common_user_region", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "User Region" + }, + { + "name": "common_device_id", + "type": "string", + "label": "Device ID" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "ISP" + }, + { + "name": "common_device_tag", + "type": "string", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "label": "Device Tag" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "label": "Encapsulation" + }, + { + "name": "common_app_label", + "type": "string", + "label": "Application Label" + }, + { + "name": "common_tunnels", + "type": "string", + "label": "Tunnels" + }, + { + "name": "common_protocol_label", + "type": "string", + "label": "Protocol Label" + }, + { + "name": "common_app_id", + "type": "string", + "label": "Application ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "type": "string", + "label": "Surrogate ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "type": "string", + "label": "L7 Protocol" + }, + { + "name": "common_service_category", + "type": { + "type": "array", + "items": "int" + }, + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "label": "FQDN Category" + }, + { + "name": "common_start_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "label": "Start Time" + }, + { + "name": "common_end_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "label": "End Time" + }, + { + "name": "common_establish_latency_ms", + "type": "long", + "label": "Establish Latency(ms)" + }, + { + "name": "common_con_duration_ms", + "type": "long", + "label": "Duration(ms)" + }, + { + "name": "common_stream_dir", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "label": "Stream Direction" + }, + { + "name": "common_address_list", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Address List" + }, + { + "name": "common_has_dup_traffic", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "label": "Duplication Traffic" + }, + { + "name": "common_stream_error", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Stream Error" + }, + { + "name": "common_stream_trace_id", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "Session ID" + }, + { + "name": "common_link_info_c2s", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(c2s)" + }, + { + "name": "common_link_info_s2c", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(s2c)" + }, + { + "name": "common_c2s_ipfrag_num", + "type": "long", + "label": "Fragmentation Packets(c2s)" + }, + { + "name": "common_s2c_ipfrag_num", + "type": "long", + "label": "Fragmentation Packets(s2c)" + }, + { + "name": "common_c2s_tcp_lostlen", + "type": "long", + "label": "Sequence Gap Loss(c2s)" + }, + { + "name": "common_s2c_tcp_lostlen", + "type": "long", + "label": "Sequence Gap Loss(s2c)" + }, + { + "name": "common_c2s_tcp_unorder_num", + "type": "long", + "label": "Unorder Packets(c2s)" + }, + { + "name": "common_s2c_tcp_unorder_num", + "type": "long", + "label": "Unorder Packets(s2c)" + }, + { + "name": "common_c2s_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(c2s)" + }, + { + "name": "common_s2c_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(s2c)" + }, + { + "name": "common_c2s_byte_retrans", + "type": "long", + "label": "Byte Retransmission(c2s)" + }, + { + "name": "common_s2c_byte_retrans", + "type": "long", + "label": "Byte Retransmission(s2c)" + }, + { + "name": "common_tcp_client_isn", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "TCP Client ISN" + }, + { + "name": "common_tcp_server_isn", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "TCP Server ISN" + }, + { + "name": "common_first_ttl", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "First TTL" + }, + { + "name": "common_processing_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "label": "Processing Time" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "http_url", + "type": "string", + "label": "HTTP.URL" + }, + { + "name": "http_host", + "type": "string", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "label": "HTTP.Host" + }, + { + "name": "http_domain", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "HTTP.Domain" + }, + { + "name": "http_request_line", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Request Line" + }, + { + "name": "http_response_line", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Response Line" + }, + { + "name": "http_request_header", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Request Headers" + }, + { + "name": "http_response_header", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Response Headers" + }, + { + "name": "http_request_content", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Request Content" + }, + { + "name": "http_request_content_length", + "label": "HTTP.Request Content Length", + "type": "string" + }, + { + "name": "http_request_content_type", + "label": "HTTP.Request Content Type", + "type": "string" + }, + { + "name": "http_response_content", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Response Content" + }, + { + "name": "http_response_content_length", + "label": "HTTP.Response Content Length", + "type": "string" + }, + { + "name": "http_response_content_type", + "label": "HTTP.Response Content Type", + "type": "string" + }, + { + "name": "http_request_body", + "type": "string", + "doc": { + "constraints": { + "type": "file" + } + }, + "label": "HTTP.Request Body" + }, + { + "name": "http_response_body", + "type": "string", + "doc": { + "constraints": { + "type": "file" + } + }, + "label": "HTTP.Response Body" + }, + { + "name": "http_request_body_key", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Request Body Key" + }, + { + "name": "http_response_body_key", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Response Body Key" + }, + { + "name": "http_proxy_flag", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Proxy Flag" + }, + { + "name": "http_sequence", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Sequence" + }, + { + "name": "http_snapshot", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Snapshot" + }, + { + "name": "http_cookie", + "type": "string", + "label": "HTTP.Cookie" + }, + { + "name": "http_referer", + "type": "string", + "label": "HTTP.Referer" + }, + { + "name": "http_user_agent", + "type": "string", + "label": "HTTP.User Agent" + }, + { + "name": "http_content_length", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Content Length" + }, + { + "name": "http_content_type", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Content Type" + }, + { + "name": "http_set_cookie", + "type": "string", + "label": "HTTP.Set Cookie" + }, + { + "name": "http_version", + "type": "string", + "label": "HTTP.Version" + }, + { + "name": "http_response_latency_ms", + "type": "long", + "label": "HTTP.Response Latency(ms)" + }, + { + "name": "http_session_duration_ms", + "type": "long", + "label": "HTTP.Session Duration(ms)" + }, + { + "name": "http_action_file_size", + "type": "int", + "label": "HTTP.Action File Size" + }, + { + "name": "dns_message_id", + "type": "int", + "label": "DNS.Message ID" + }, + { + "name": "dns_qr", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "label": "DNS.QR" + }, + { + "name": "dns_opcode", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "label": "DNS.OPCODE" + }, + { + "name": "dns_aa", + "type": "int", + "label": "DNS.AA" + }, + { + "name": "dns_tc", + "type": "int", + "label": "DNS.TC" + }, + { + "name": "dns_rd", + "type": "int", + "label": "DNS.RD" + }, + { + "name": "dns_ra", + "type": "int", + "label": "DNS.RA" + }, + { + "name": "dns_rcode", + "type": "int", + "label": "DNS.RCODE" + }, + { + "name": "dns_qdcount", + "type": "int", + "label": "DNS.QDCOUNT" + }, + { + "name": "dns_ancount", + "type": "int", + "label": "DNS.ANCOUNT" + }, + { + "name": "dns_nscount", + "type": "int", + "label": "DNS.NSCOUNT" + }, + { + "name": "dns_arcount", + "type": "int", + "label": "DNS.ARCOUNT" + }, + { + "name": "dns_qname", + "type": "string", + "label": "DNS.QNAME" + }, + { + "name": "dns_qtype", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "label": "DNS.QTYPE" + }, + { + "name": "dns_qclass", + "type": "int", + "label": "DNS.QCLASS" + }, + { + "name": "dns_cname", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "DNS.CNAME" + }, + { + "name": "dns_sub", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "label": "DNS.SUB" + }, + { + "name": "dns_rr", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "DNS.RR" + }, + { + "name": "sip_call_id", + "type": "string", + "label": "SIP.Call-ID" + }, + { + "name": "sip_originator_description", + "type": "string", + "label": "SIP.Originator" + }, + { + "name": "sip_responder_description", + "type": "string", + "label": "SIP.Responder" + }, + { + "name": "sip_user_agent", + "type": "string", + "label": "SIP.User-Agent" + }, + { + "name": "sip_server", + "type": "string", + "label": "SIP.Server" + }, + { + "name": "sip_originator_sdp_connect_ip", + "type": "string", + "label": "SIP.Originator IP" + }, + { + "name": "sip_originator_sdp_media_port", + "type": "int", + "label": "SIP.Originator Port" + }, + { + "name": "sip_originator_sdp_media_type", + "type": "string", + "label": "SIP.Originator Media Type" + }, + { + "name": "sip_originator_sdp_content", + "type": "string", + "label": "SIP.Originator Content" + }, + { + "name": "sip_responder_sdp_connect_ip", + "type": "string", + "label": "SIP.Responder IP" + }, + { + "name": "sip_responder_sdp_media_port", + "type": "int", + "label": "SIP.Responder Port" + }, + { + "name": "sip_responder_sdp_media_type", + "type": "string", + "label": "SIP.Responder Media Type" + }, + { + "name": "sip_responder_sdp_content", + "type": "string", + "label": "SIP.Responder Content" + }, + { + "name": "sip_duration", + "type": "int", + "label": "SIP.Duration" + }, + { + "name": "sip_bye", + "type": "string", + "label": "SIP.Bye" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/voip_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/schema/voip_record.json new file mode 100644 index 0000000..a3748c8 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/schema/voip_record.json @@ -0,0 +1,1384 @@ +{ + "type": "record", + "name": "voip_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_device_group", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_device_group", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_device_group", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_mirrored_pkts", + "common_mirrored_bytes" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "SIP": { + "$ref": "public_schema_info.json#/schema_type/SIP" + }, + "RTP": { + "$ref": "public_schema_info.json#/schema_type/RTP" + }, + "VoIP": { + "$ref": "public_schema_info.json#/schema_type/VoIP" + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "internal_columns": [ + "common_recv_time", + "common_log_id", + "common_processing_time" + ], + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "VoIP", + "value": "VoIP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_encapsulation/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data" + }, + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "files" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ] + }, + "type": "int" + } + ] +} \ No newline at end of file