gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

21.12 update

Browse Source
This commit is contained in:
zhanghongqing
2021-12-21 17:04:24 +08:00
parent b873773407
commit f4cee9dd8c
79 changed files with 3655 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
TSG发布版本更新记录/TSG-21.12/druid/update-druid-2112.sql → TSG发布版本更新记录/TSG-21.12/druid/README.txt
View File

@@ -1,20 +1,54 @@
#更新一 traffic_app_stat_log
# 更新的时候，先停掉之前的任务，然后执行sql，最后再提交新任务
druid 任务更新步骤
概述
1.先停掉之前的任务，然后执行sql，最后再提交新任务
2.根据实际情况执行集群或单机版操作
操作
一.更新traffic_app_stat_log
1.登录druid提交任务服务器（通常为第一台）进入 */druid_topology/rule/
2.停止需要更新的任务 ./supervisor-manger terminate traffic_app_stat_log ， ./supervisor-manger terminate traffic_metrics_log
3. 修改traffic_app_stat_log.json traffic_metrics_log.json中IP地址后，复制到 */druid_topology/tasks下
4.登录galaxy使用的mariadb，使用druid数据库执行以下SQL
执行SQL一：
DELETE FROM druid.druid_pendingsegments WHERE dataSource = 'traffic_app_stat_log';
DELETE FROM druid.druid_datasource WHERE dataSource = 'traffic_app_stat_log';
#更新二 traffic_metrics_log(不要在widows环境解压操作)
执行SQL二：
单机版sql:
INSERT INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES('traffic_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-12-12T12:32:13.207Z', 'traffic_metrics_log', '2021-12-12T12:32:14.239Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-12-12T12:32:13.207Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6D6574726963735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D31322D31325431323A33323A31332E3230375A222C226C6F616453706563223A7B2274797065223A226C6F63616C222C2270617468223A222F6170616368652D64727569642D302E31382E312F7661722F64727569642F7365676D656E74732F747261666669635F6D6574726963735F6C6F672F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032312D31322D31325431323A33323A31332E3230375A2F302F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226465766963655F69642C616C6C6F775F636F6E6E5F6E756D2C616C6C6F775F696E5F62797465732C616C6C6F775F696E5F7061636B6574732C616C6C6F775F6F75745F62797465732C616C6C6F775F6F75745F7061636B6574732C636C6F73655F636F6E6E5F6E756D2C64656661756C745F636F6E6E5F6E756D2C64656661756C745F696E5F62797465732C64656661756C745F696E5F7061636B6574732C64656661756C745F6F75745F62797465732C64656661756C745F6F75745F7061636B6574732C64656E795F636F6E6E5F6E756D2C64656E795F696E5F62797465732C64656E795F696E5F7061636B6574732C64656E795F6F75745F62797465732C64656E795F6F75745F7061636B6574732C65737461626C69736865645F636F6E6E5F6E756D2C696E746572636570745F636F6E6E5F6E756D2C696E746572636570745F696E5F62797465732C696E746572636570745F696E5F7061636B6574732C696E746572636570745F6F75745F62797465732C696E746572636570745F6F75745F7061636B6574732C6D6F6E69746F725F636F6E6E5F6E756D2C6D6F6E69746F725F696E5F62797465732C6D6F6E69746F725F696E5F7061636B6574732C6D6F6E69746F725F6F75745F62797465732C6D6F6E69746F725F6F75745F7061636B6574732C6E65775F636F6E6E5F6E756D2C746F74616C5F696E5F62797465732C746F74616C5F696E5F7061636B6574732C746F74616C5F6F75745F62797465732C656E7472616E63655F69642C746F74616C5F6F75745F7061636B6574732C70696E6E696E675F6E756D2C6E6F745F70696E6E696E675F6E756D2C6D617962655F70696E6E696E675F6E756D2C616C6572745F62797465732C696E735F62797465732C68696A6B5F62797465732C61645F7265666C656374696F6E5F62797465732C61645F666C6F6F645F62797465732C61645F63635F62797465732C696E7463705F6D6F6E5F6E756D2C696E7463705F616C6C6F775F6E756D2C696E7463705F64656E795F6E756D2C696E7463705F72646972745F6E756D2C696E7463705F7265706C5F6E756D2C696E7463705F68696A6B5F6E756D2C696E7463705F696E735F6E756D2C696E7463705F656469745F656C656D5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A31313633342C226964656E746966696572223A22747261666669635F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D31322D31325431323A33323A31332E3230375A227D);
单机版文件：standalone/traffic_metrics_log.zip
集群版sql:
INSERT INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES('traffic_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-12-13T02:46:13.726Z', 'traffic_metrics_log', '2021-12-13T02:46:14.984Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-12-13T02:46:13.726Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6D6574726963735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D31322D31335430323A34363A31332E3732365A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F6D6574726963735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D31322D31335430325F34365F31332E3732365A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226465766963655F69642C616C6C6F775F636F6E6E5F6E756D2C616C6C6F775F696E5F62797465732C616C6C6F775F696E5F7061636B6574732C616C6C6F775F6F75745F62797465732C616C6C6F775F6F75745F7061636B6574732C636C6F73655F636F6E6E5F6E756D2C64656661756C745F636F6E6E5F6E756D2C64656661756C745F696E5F62797465732C64656661756C745F696E5F7061636B6574732C64656661756C745F6F75745F62797465732C64656661756C745F6F75745F7061636B6574732C64656E795F636F6E6E5F6E756D2C64656E795F696E5F62797465732C64656E795F696E5F7061636B6574732C64656E795F6F75745F62797465732C64656E795F6F75745F7061636B6574732C65737461626C69736865645F636F6E6E5F6E756D2C696E746572636570745F636F6E6E5F6E756D2C696E746572636570745F696E5F62797465732C696E746572636570745F696E5F7061636B6574732C696E746572636570745F6F75745F62797465732C696E746572636570745F6F75745F7061636B6574732C6D6F6E69746F725F636F6E6E5F6E756D2C6D6F6E69746F725F696E5F62797465732C6D6F6E69746F725F696E5F7061636B6574732C6D6F6E69746F725F6F75745F62797465732C6D6F6E69746F725F6F75745F7061636B6574732C6E65775F636F6E6E5F6E756D2C746F74616C5F696E5F62797465732C746F74616C5F696E5F7061636B6574732C746F74616C5F6F75745F62797465732C656E7472616E63655F69642C746F74616C5F6F75745F7061636B6574732C70696E6E696E675F6E756D2C6E6F745F70696E6E696E675F6E756D2C6D617962655F70696E6E696E675F6E756D2C616C6572745F62797465732C696E735F62797465732C68696A6B5F62797465732C61645F7265666C656374696F6E5F62797465732C61645F666C6F6F645F62797465732C61645F63635F62797465732C696E7463705F6D6F6E5F6E756D2C696E7463705F616C6C6F775F6E756D2C696E7463705F64656E795F6E756D2C696E7463705F72646972745F6E756D2C696E7463705F7265706C5F6E756D2C696E7463705F68696A6B5F6E756D2C696E7463705F696E735F6E756D2C696E7463705F656469745F656C656D5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A31313633342C226964656E746966696572223A22747261666669635F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D31322D31335430323A34363A31332E3732365A227D);
集群版文件路径： cluster/traffic_metrics_log.zip
5.更新索引文件
#解压后替换druid目录下的文件(不要在widows环境解压操作)
单机版文件：standalone/traffic_metrics_log.zip 解压后替换druid目录下的文件var/druid/segments/
#集群版需要上传到hdfs
集群版文件： cluster/traffic_metrics_log.zip 解压到hdfs服务器上后运行以下命令
hdfs dfs -put traffic_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2021-12-13T02_46_13.726Z/ /druid/segments/traffic_metrics_log/30000101T000000.000Z_30000102T000000.000Z
6.启动任务，druid服务器上
druid_topology/目录下
使用以下命令提交
变量：${var} traffic_app_stat_log.json，traffic_metrics_log.json
变量：${druid_host} druid服务器地址
curl -X 'POST' -H 'Content-Type:application/json' -d @${var} http://$druid_host:8081/druid/indexer/v1/supervisor
7.修复log4j2漏洞
集群版：
将log4j2.component.properties文件放在所有druid安装目录conf/druid/cluster/_common/
单机版：
将log4j2.component.properties文件放在conf/druid/single-server/small|medium|large/_common/
*需要重启所有druid服务
8.验证
执行状态为RUNNING则任务正常
curl http://${druid_host}:8089/druid/indexer/v1/supervisor?state=true |jq
9.任务异常处理
1.登录druid提交任务服务器（通常为第一台）进入 */druid_topology/rule/
2. 执行./supervisor-manger reset 任务名称 ，例如./supervisor-manger reset traffic_app_stat_log

1
TSG发布版本更新记录/TSG-21.12/druid/log4j2.component.properties Normal file
View File

@@ -0,0 +1 @@
log4j2.formatMsgNoLookups=true

24
TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties
View File

@@ -1,24 +0,0 @@
#管理kafka地址
input.kafka.servers=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
#input.kafka.servers=192.168.44.12:9094
#hbase zookeeper地址 用于连接HBase
hbase.zookeeper.servers=192.168.44.11,192.168.44.14,192.168.44.15
#hbase.zookeeper.servers=192.168.44.11:2181
#--------------------------------Kafka消费组信息------------------------------#
#kafka 接收数据topic
input.kafka.topic=RADIUS-RECORD
#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
group.id=radius-flink-20211124
#--------------------------------topology配置------------------------------#
#ip-account对应关系表
hbase.framedip.table.name=tsg_galaxy:relation_framedip_account
#定位库地址
tools.library=/home/bigdata/topology/dat/
#account-ip对应关系表
hbase.account.table.name=tsg_galaxy:relation_account_framedip

3
TSG发布版本更新记录/TSG-21.12/flink/config.sh Normal file
View File

@@ -0,0 +1,3 @@
#flink bin/config.sh中临时目录设置，用于存放进程id
DEFAULT_ENV_PID_DIR="$(cd "`dirname "$0"`"/..; pwd)/tmp"

5
TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties
View File

@@ -1,5 +0,0 @@
# dos任务新增以下配置
#baseline ttl，单位：天
hbase.baseline.ttl=30

1
TSG发布版本更新记录/TSG-21.12/flink/log4j2.component.properties Normal file
View File

@@ -0,0 +1 @@
log4j2.formatMsgNoLookups=true

38
TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties
View File

@@ -1,38 +0,0 @@
#kafka的地址信息
source.kafka.broker=192.168.44.11:9094
source.kafka.group.id =vpn-1206-1
source.kafka.topic=SESSION-RECORD-COMPLETED
source.kafka.parallelism=1
max.poll.records=3000
session.timeout.ms=60000
max.partition.fetch.bytes=31457280
#hbase的zk地址
zk.host=192.168.44.11:2181
#写入hbase并行度
sink.hbase.parallelism=1
#写入hbase列簇
sink.hbase.fm=common
#写入hbase表名
sink.hbase.table=tsg_galaxy:recommendation_app_cip
#任务并行度
task.parallelism=1
#窗口延迟等待时间单位秒
watermark.time=1
#top结果限制
top.limit=10000
#滑动窗口总时间单位分钟
slidingwindow.time.minute=30
#每个滑块时间单位分钟
slidingwindowslot.time.minute=1
#kafka是否开启安全验证 0不开启 1SSL 2 SASL
kafka.security=2
#kafka SASL验证用户名
kafka.user=admin
#kafka SASL及SSL验证密码
kafka.pin=galaxy2019
#1SSL需要
tools.library=D:\\K18-Phase2\\tsgSpace\\dat\\tsg\\
#是否接受全量app 无过滤条件false 白名单过滤true
has.filter=false
#只计算filter命中的common_app_label逗号分隔 baidu.com,qq 可不填写
app.white.list=

11
TSG发布版本更新记录/TSG-21.12/hbase/update-hbase.txt
View File

@@ -1,3 +1,10 @@
-- hbase新增表
-- hbase新增表 在hbase服务器，使用hbase shell命令
create 'tsg_galaxy:recommendation_app_cip', {NAME => 'common', VERSIONS => 1}
create 'tsg_galaxy:recommendation_app_cip', {NAME => 'common', VERSIONS => 1}
-- hbase 新增字段
disable "tsg_galaxy:job_result"
alter "tsg_galaxy:job_result",NAME=>'detail',TTL=>'1800'
alter "tsg_galaxy:job_result",NAME=>'result',TTL=>'1800'
enable "tsg_galaxy:job_result"

250
TSG发布版本更新记录/TSG-21.12/hbase/update-phoenix.txt
View File

@@ -1,11 +1,257 @@
#进入目录 phoenix-hbase-2.2-5.1.2-bin/bin 执行./sqlline.py 后分别执行以下建表语句
-- phoenix 新增字段:
alter view "tsg_galaxy"."relation_account_framedip" add "radius"."acct_status_type" UNSIGNED_INT;
-- Phoenix新增表
CREATE view "tsg_galaxy"."recommendation_app_cip"(
ROWKEY VARCHAR PRIMARY KEY,
"common"."app_label" VARCHAR,
"common"."client_ip_list" VARCHAR,
"common"."last_update_time" UNSIGNED_LONG);
"common"."last_update_time" UNSIGNED_LONG);
-- Phoenix新增表
CREATE schema IF NOT EXISTS "tsg_galaxy";
CREATE table IF NOT EXISTS "tsg_galaxy"."job_result"(
ROWKEY VARCHAR PRIMARY KEY,
"detail"."is_done" BOOLEAN,
"detail"."done_progress" UNSIGNED_FLOAT,
"detail"."is_canceled" BOOLEAN,
"detail"."discovery_field" VARCHAR,
"detail"."last_query_time" UNSIGNED_LONG,
"detail"."count" UNSIGNED_LONG,
"result"."app_extra_info" VARCHAR,
"result"."attack_type" VARCHAR,
"result"."bit_rate" VARCHAR,
"result"."common_action" VARCHAR,
"result"."common_address_type" VARCHAR,
"result"."common_app_label" VARCHAR,
"result"."common_c2s_byte_diff" VARCHAR,
"result"."common_c2s_byte_num" VARCHAR,
"result"."common_c2s_byte_retrans" VARCHAR,
"result"."common_c2s_ipfrag_num" VARCHAR,
"result"."common_c2s_pkt_diff" VARCHAR,
"result"."common_c2s_pkt_num" VARCHAR,
"result"."common_c2s_pkt_retrans" VARCHAR,
"result"."common_c2s_tcp_lostlen" VARCHAR,
"result"."common_c2s_tcp_unorder_num" VARCHAR,
"result"."common_client_asn" VARCHAR,
"result"."common_client_ip" VARCHAR,
"result"."common_client_location" VARCHAR,
"result"."common_client_port" VARCHAR,
"result"."common_con_duration_ms" VARCHAR,
"result"."common_data_center" VARCHAR,
"result"."common_device_group" VARCHAR,
"result"."common_device_id" VARCHAR,
"result"."common_direction" VARCHAR,
"result"."common_end_time" VARCHAR,
"result"."common_establish_latency_ms" VARCHAR,
"result"."common_external_ip" VARCHAR,
"result"."common_imei" VARCHAR,
"result"."common_imsi" VARCHAR,
"result"."common_internal_ip" VARCHAR,
"result"."common_l4_protocol" VARCHAR,
"result"."common_l7_protocol" VARCHAR,
"result"."common_mirrored_bytes" VARCHAR,
"result"."common_mirrored_pkts" VARCHAR,
"result"."common_phone_number" VARCHAR,
"result"."common_policy_id" VARCHAR,
"result"."common_protocol_label" VARCHAR,
"result"."common_s2c_byte_diff" VARCHAR,
"result"."common_s2c_byte_num" VARCHAR,
"result"."common_s2c_byte_retrans" VARCHAR,
"result"."common_s2c_ipfrag_num" VARCHAR,
"result"."common_s2c_pkt_diff" VARCHAR,
"result"."common_s2c_pkt_num" VARCHAR,
"result"."common_s2c_pkt_retrans" VARCHAR,
"result"."common_s2c_tcp_lostlen" VARCHAR,
"result"."common_s2c_tcp_unorder_num" VARCHAR,
"result"."common_schema_type" VARCHAR,
"result"."common_server_asn" VARCHAR,
"result"."common_server_ip" VARCHAR,
"result"."common_server_location" VARCHAR,
"result"."common_server_port" VARCHAR,
"result"."common_service_category" VARCHAR,
"result"."common_sessions" VARCHAR,
"result"."common_sled_ip" VARCHAR,
"result"."common_start_time" VARCHAR,
"result"."common_stream_dir" VARCHAR,
"result"."common_stream_error" VARCHAR,
"result"."common_stream_trace_id" VARCHAR,
"result"."common_sub_action" VARCHAR,
"result"."common_subscriber_id" VARCHAR,
"result"."common_tcp_client_isn" VARCHAR,
"result"."common_tcp_server_isn" VARCHAR,
"result"."common_tunnels" VARCHAR,
"result"."common_userdefine_app_name" VARCHAR,
"result"."conditions" VARCHAR,
"result"."destination_country" VARCHAR,
"result"."destination_ip" VARCHAR,
"result"."dns_aa" VARCHAR,
"result"."dns_ancount" VARCHAR,
"result"."dns_arcount" VARCHAR,
"result"."dns_message_id" VARCHAR,
"result"."dns_nscount" VARCHAR,
"result"."dns_opcode" VARCHAR,
"result"."dns_qclass" VARCHAR,
"result"."dns_qdcount" VARCHAR,
"result"."dns_qname" VARCHAR,
"result"."dns_qr" VARCHAR,
"result"."dns_qtype" VARCHAR,
"result"."dns_ra" VARCHAR,
"result"."dns_rcode" VARCHAR,
"result"."dns_rd" VARCHAR,
"result"."dns_sub" VARCHAR,
"result"."dns_tc" VARCHAR,
"result"."doh_aa" VARCHAR,
"result"."doh_ancount" VARCHAR,
"result"."doh_arcount" VARCHAR,
"result"."doh_cname" VARCHAR,
"result"."doh_cookie" VARCHAR,
"result"."doh_host" VARCHAR,
"result"."doh_message_id" VARCHAR,
"result"."doh_nscount" VARCHAR,
"result"."doh_opcode" VARCHAR,
"result"."doh_qclass" VARCHAR,
"result"."doh_qdcount" VARCHAR,
"result"."doh_qname" VARCHAR,
"result"."doh_qr" VARCHAR,
"result"."doh_qtype" VARCHAR,
"result"."doh_ra" VARCHAR,
"result"."doh_rcode" VARCHAR,
"result"."doh_rd" VARCHAR,
"result"."doh_referer" VARCHAR,
"result"."doh_rr" VARCHAR,
"result"."doh_sub" VARCHAR,
"result"."doh_tc" VARCHAR,
"result"."doh_url" VARCHAR,
"result"."doh_user_agent" VARCHAR,
"result"."doh_version" VARCHAR,
"result"."ftp_account" VARCHAR,
"result"."ftp_content" VARCHAR,
"result"."ftp_link_type" VARCHAR,
"result"."ftp_url" VARCHAR,
"result"."gtp_apn" VARCHAR,
"result"."gtp_downlink_teid" VARCHAR,
"result"."gtp_end_user_ipv4" VARCHAR,
"result"."gtp_end_user_ipv6" VARCHAR,
"result"."gtp_imei" VARCHAR,
"result"."gtp_imsi" VARCHAR,
"result"."gtp_msg_type" VARCHAR,
"result"."gtp_phone_number" VARCHAR,
"result"."gtp_uplink_teid" VARCHAR,
"result"."gtp_version" VARCHAR,
"result"."http_action_file_size" VARCHAR,
"result"."http_cookie" VARCHAR,
"result"."http_domain" VARCHAR,
"result"."http_host" VARCHAR,
"result"."http_referer" VARCHAR,
"result"."http_request_body" VARCHAR,
"result"."http_request_content_length" VARCHAR,
"result"."http_request_content_type" VARCHAR,
"result"."http_request_header" VARCHAR,
"result"."http_response_body" VARCHAR,
"result"."http_response_content_length" VARCHAR,
"result"."http_response_content_type" VARCHAR,
"result"."http_response_header" VARCHAR,
"result"."http_response_latency_ms" VARCHAR,
"result"."http_session_duration_ms" VARCHAR,
"result"."http_set_cookie" VARCHAR,
"result"."http_url" VARCHAR,
"result"."http_user_agent" VARCHAR,
"result"."http_version" VARCHAR,
"result"."mail_account" VARCHAR,
"result"."mail_attachment_name" VARCHAR,
"result"."mail_bcc" VARCHAR,
"result"."mail_cc" VARCHAR,
"result"."mail_eml_file" VARCHAR,
"result"."mail_from" VARCHAR,
"result"."mail_from_cmd" VARCHAR,
"result"."mail_protocol_type" VARCHAR,
"result"."mail_subject" VARCHAR,
"result"."mail_to" VARCHAR,
"result"."mail_to_cmd" VARCHAR,
"result"."packet_rate" VARCHAR,
"result"."quic_sni" VARCHAR,
"result"."quic_user_agent" VARCHAR,
"result"."quic_version" VARCHAR,
"result"."radius_account" VARCHAR,
"result"."radius_acct_authentic" VARCHAR,
"result"."radius_acct_delay_time" VARCHAR,
"result"."radius_acct_input_octets" VARCHAR,
"result"."radius_acct_input_packets" VARCHAR,
"result"."radius_acct_interim_interval" VARCHAR,
"result"."radius_acct_link_count" VARCHAR,
"result"."radius_acct_multi_session_id" VARCHAR,
"result"."radius_acct_output_octets" VARCHAR,
"result"."radius_acct_output_packets" VARCHAR,
"result"."radius_acct_session_id" VARCHAR,
"result"."radius_acct_session_time" VARCHAR,
"result"."radius_acct_status_type" VARCHAR,
"result"."radius_acct_terminate_cause" VARCHAR,
"result"."radius_callback_id" VARCHAR,
"result"."radius_callback_number" VARCHAR,
"result"."radius_called_station_id" VARCHAR,
"result"."radius_calling_station_id" VARCHAR,
"result"."radius_event_timestamp" VARCHAR,
"result"."radius_framed_ip" VARCHAR,
"result"."radius_framed_protocol" VARCHAR,
"result"."radius_idle_timeout" VARCHAR,
"result"."radius_nas_ip" VARCHAR,
"result"."radius_nas_port" VARCHAR,
"result"."radius_packet_type" VARCHAR,
"result"."radius_service_type" VARCHAR,
"result"."radius_session_timeout" VARCHAR,
"result"."radius_termination_action" VARCHAR,
"result"."rtp_originator_dir" VARCHAR,
"result"."rtp_payload_type_c2s" VARCHAR,
"result"."rtp_payload_type_s2c" VARCHAR,
"result"."rtp_pcap_path" VARCHAR,
"result"."session_rate" VARCHAR,
"result"."severity" VARCHAR,
"result"."sip_bye" VARCHAR,
"result"."sip_call_id" VARCHAR,
"result"."sip_duration" VARCHAR,
"result"."sip_originator_description" VARCHAR,
"result"."sip_originator_sdp_connect_ip" VARCHAR,
"result"."sip_originator_sdp_content" VARCHAR,
"result"."sip_originator_sdp_media_port" VARCHAR,
"result"."sip_originator_sdp_media_type" VARCHAR,
"result"."sip_responder_description" VARCHAR,
"result"."sip_responder_sdp_connect_ip" VARCHAR,
"result"."sip_responder_sdp_content" VARCHAR,
"result"."sip_responder_sdp_media_port" VARCHAR,
"result"."sip_responder_sdp_media_type" VARCHAR,
"result"."sip_server" VARCHAR,
"result"."sip_user_agent" VARCHAR,
"result"."source_country_list" VARCHAR,
"result"."source_ip_list" VARCHAR,
"result"."ssh_auth_success" VARCHAR,
"result"."ssh_cipher_alg" VARCHAR,
"result"."ssh_client_version" VARCHAR,
"result"."ssh_compression_alg" VARCHAR,
"result"."ssh_hassh" VARCHAR,
"result"."ssh_host_key" VARCHAR,
"result"."ssh_host_key_alg" VARCHAR,
"result"."ssh_kex_alg" VARCHAR,
"result"."ssh_mac_alg" VARCHAR,
"result"."ssh_server_version" VARCHAR,
"result"."ssh_version" VARCHAR,
"result"."ssl_cert_issuer" VARCHAR,
"result"."ssl_cert_subject" VARCHAR,
"result"."ssl_cert_verify" VARCHAR,
"result"."ssl_client_side_latency" VARCHAR,
"result"."ssl_client_side_version" VARCHAR,
"result"."ssl_cn" VARCHAR,
"result"."ssl_con_latency_ms" VARCHAR,
"result"."ssl_error" VARCHAR,
"result"."ssl_intercept_state" VARCHAR,
"result"."ssl_ja3_hash" VARCHAR,
"result"."ssl_pinningst" VARCHAR,
"result"."ssl_san" VARCHAR,
"result"."ssl_server_side_latency" VARCHAR,
"result"."ssl_server_side_version" VARCHAR,
"result"."ssl_sni" VARCHAR);

2
TSG发布版本更新记录/TSG-21.12/kafka/add-topic.txt
View File

@@ -1,4 +1,4 @@
#新增kafka topic
#新增kafka topic，在kafka服务器上运行以下命令，IP地址为zookeeper地址 分区副本以实际为准
kafka-topics.sh '--create' '--zookeeper' 'IP:2181/kafka' '--replication-factor' 1 '--partitions' 1 '--topic' TRAFFIC-APP-STAT

6
TSG发布版本更新记录/TSG-21.12/qgw/README.txt Normal file
View File

@@ -0,0 +1,6 @@
更新配置文件步骤
1.修改backup_config.sh脚本中的配置后执行
2.将备份中文件galaxy-qgw-service.yml复制一份放入更新目录galaxy-qgw-service/config/
3.修改galaxy-qgw-service.yml增加更新文件update-galaxy-qgw-service.yml中的配置项
4.修改push_config.sh 配置项后执行提交配置到nacos

31
TSG发布版本更新记录/TSG-21.12/qgw/backup_config.sh Normal file
View File

@@ -0,0 +1,31 @@
#!/bin/bash
nacos_server="192.168.44.12"
tenant="prod"
group_id="Galaxy"
app_name="galaxy-qgw-service"
username="nacos"
password="nacos"
#放置备份配置的目录
backup_config_path=/home/tmp/21.11
search_config_url="http://$nacos_server:8848/nacos/v1/cs/configs?dataId=&group=${group_id}&appName=$app_name&username=$username&password=$password&tenant=$tenant&search=accurate&pageNo=1&pageSize=10000"
backup_config_ids=$(curl -X GET $search_config_url|jq -r .pageItems[].dataId)
array=(${backup_config_ids//,/ })
if [ ! -d $backup_config_path ];then
mkdir -p $backup_config_path
fi
for data_id in ${array[@]}
do
config_url="http://$nacos_server:8848/nacos/v1/cs/configs?dataId=$data_id&group=$group_id&appName=$app_name&username=$username&password=$password&tenant=$tenant"
#备份
curl $config_url > $backup_config_path/$data_id
done
#`curl -X DELETE $config_url`

322
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/active_defence_event.json Normal file
View File

@@ -0,0 +1,322 @@
{
"type": "record",
"name": "active_defence_event",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"partition_key": "common_recv_time",
"schema_query": {
"dimensions": [
"common_policy_id",
"ad_target_ip",
"ad_cc_target_url"
],
"metrics": [
"ad_target_ip",
"ad_sent_byte_num",
"ad_sent_pkt_num",
"ad_cc_initiate_connection_num",
"ad_cc_established_connection_num",
"ad_cc_rejected_connection_num"
],
"filters": [
"common_policy_id",
"ad_target_ip",
"ad_target_port",
"ad_protocol",
"common_address_type",
"ad_sent_byte_num",
"ad_sent_pkt_num",
"ad_cc_initiate_connection_num",
"ad_cc_established_connection_num",
"ad_cc_rejected_connection_num"
]
},
"schema_type": {
"REFLECTION": {
"columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"common_address_type",
"common_device_id",
"common_egress_link_id",
"common_ingress_link_id",
"common_entrance_id",
"common_user_region",
"ad_method",
"ad_protocol",
"ad_target_ip",
"ad_target_port",
"ad_target_ip_location",
"ad_target_ip_asn",
"ad_reflector_profile_id",
"ad_sent_pkt_num",
"ad_sent_byte_num",
"ad_generate_time"
],
"default_columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"ad_target_ip",
"ad_target_port",
"ad_reflector_profile_id",
"ad_sent_pkt_num",
"ad_sent_byte_num"
]
},
"FLOOD": {
"columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"common_address_type",
"common_device_id",
"common_egress_link_id",
"common_ingress_link_id",
"common_entrance_id",
"common_user_region",
"ad_method",
"ad_protocol",
"ad_target_ip",
"ad_target_port",
"ad_target_ip_location",
"ad_target_ip_asn",
"ad_claimed_src_ip_profile_id",
"ad_sent_pkt_num",
"ad_sent_byte_num",
"ad_generate_time"
],
"default_columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"ad_target_ip",
"ad_target_port",
"ad_claimed_src_ip_profile_id",
"ad_protocol"
]
},
"CC": {
"columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"common_address_type",
"common_device_id",
"common_egress_link_id",
"common_ingress_link_id",
"common_entrance_id",
"common_user_region",
"ad_method",
"ad_protocol",
"ad_cc_target_url",
"ad_claimed_src_ip_profile_id",
"ad_cc_initiate_connection_num",
"ad_cc_established_connection_num",
"ad_cc_rejected_connection_num",
"ad_generate_time"
],
"default_columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"ad_cc_target_url",
"ad_claimed_src_ip_profile_id",
"ad_protocol"
]
}
},
"default_columns": [
"common_recv_time",
"common_log_id",
"common_policy_id",
"ad_target_ip",
"ad_target_port",
"ad_cc_target_url"
]
},
"fields": [
{
"name": "common_recv_time",
"label": "Receive Time",
"doc": {
"allow_query": "true",
"constraints": {
"type": "timestamp"
},
"format": {
"functions": "current_timestamp"
}
},
"type": "long"
},
{
"name": "common_log_id",
"label": "Log ID",
"doc": {
"allow_query": "true",
"format": {
"functions": "snowflake_id"
}
},
"type": "long"
},
{
"name": "common_policy_id",
"label": "Policy ID",
"doc": {
"allow_query": "true"
},
"type": "long"
},
{
"name": "common_address_type",
"label": "Address Type",
"doc": {
"data": [
{
"code": "4",
"value": "ipv4"
},
{
"code": "6",
"value": "ipv6"
}
]
},
"type": "int"
},
{
"name": "common_entrance_id",
"label": "Entrance ID",
"doc": {
"visibility": "disabled"
},
"type": "int"
},
{
"name": "common_device_id",
"label": "Device ID",
"type": "string"
},
{
"name": "common_egress_link_id",
"label": "Egress Link ID",
"doc": {
"visibility": "hidden"
},
"type": "int"
},
{
"name": "common_ingress_link_id",
"label": "Ingress Link ID",
"doc": {
"visibility": "hidden"
},
"type": "int"
},
{
"name": "common_user_region",
"label": "User Region",
"doc": {
"visibility": "hidden"
},
"type": "string"
},
{
"name": "ad_target_ip",
"label": "Target IP",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
},
"format": {
"functions": "geo_ip_country,geo_asn",
"appendTo": "ad_target_ip_location,ad_target_ip_asn"
}
},
"type": "string"
},
{
"name": "ad_target_port",
"label": "Target Port",
"type": "int"
},
{
"name": "ad_cc_target_url",
"label": "Target URL",
"doc": {
"allow_query": "true"
},
"type": "string"
},
{
"name": "ad_target_ip_location",
"label": "Target Location",
"type": "string"
},
{
"name": "ad_target_ip_asn",
"label": "Target ASN",
"type": "string"
},
{
"name": "ad_protocol",
"label": "Protocol",
"type": "string"
},
{
"name": "ad_method",
"label": "Method",
"type": "string"
},
{
"name": "ad_claimed_src_ip_profile_id",
"label": "Claimed Profile ID",
"type": "int"
},
{
"name": "ad_reflector_profile_id",
"label": "Reflector Profile ID",
"type": "int"
},
{
"name": "ad_sent_pkt_num",
"label": "Packets Sent",
"type": "int"
},
{
"name": "ad_sent_byte_num",
"label": "Bytes Sent",
"type": "int"
},
{
"name": "ad_cc_initiate_connection_num",
"label": "Initiate Numbers",
"type": "int"
},
{
"name": "ad_cc_established_connection_num",
"label": "Established Numbers",
"type": "int"
},
{
"name": "ad_cc_rejected_connection_num",
"label": "Rejected Numbers",
"type": "int"
},
{
"name": "ad_generate_time",
"label": "Generate Time",
"doc": {
"constraints": {
"type": "timestamp"
}
},
"type": "int"
}
]
}

99
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-filter.json Normal file
View File

@@ -0,0 +1,99 @@
{
"version": "1.0",
"name": "ClickHouse-Raw",
"namespace": "ClickHouse",
"filters": [
{
"name":"@start",
"value": "'2021-10-19 10:00:00'"
},
{
"name":"@end",
"value": "'2021-10-20 11:00:00'"
},
{
"name":"@common_filter",
"value": [
"common_log_id=1153021139190754263",
"common_client_ip='118.180.48.74'",
"common_client_ip='120.242.132.200'",
"common_internal_ip='223.116.37.192'",
"common_server_ip='8.8.8.8'",
"common_server_ip='114.114.114.114'",
"common_server_ip!='114.114.114.114'",
"common_server_ip='120.239.72.226'",
"common_external_ip='111.10.53.14'",
"common_client_port=52607",
"common_server_port=443",
"common_c2s_pkt_num>5",
"common_s2c_pkt_num>5",
"common_c2s_byte_num>100",
"common_s2c_byte_num<200",
"common_schema_type='DNS'",
"common_establish_latency_ms>200",
"common_con_duration_ms>10000",
"common_stream_trace_id=1153021139190754263",
"common_tcp_client_isn=2857077935",
"common_tcp_server_isn=0",
"http_domain='qq.com'",
"http_domain!='qq.com'",
"http_domain='yunser.com'",
"mail_account='abc@xx.com'",
"mail_subject='test'",
"dns_qname='qbwup.imtt.qq.com'",
"ssl_sni='mmbiz.qpic.cn'",
"ssl_sni='openai.qq.com'",
"ssl_con_latency_ms>100",
"ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'",
"common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'",
"common_server_ip='111.10.53.14' and common_server_port=443",
"common_server_ip like '120.239%'",
"common_server_ip not like '120.239%'",
"common_server_ip like '%114.114%'",
"mail_account like 'abc@%'",
"http_domain like '%baidu.com%'",
"ssl_sni like '%google.com'",
"http_domain like 'baidu%'",
"http_domain like '%baidu.com%'",
"common_client_ip in ('120.239.72.226','114.114.114.114')",
"common_client_ip not in ('120.239.72.226','114.114.114.114')",
"common_server_ip='116.177.248.126' and notEmpty(http_domain)",
"common_server_ip='116.177.248.126' and common_client_ip='120.242.132.200'",
"common_server_ip='116.177.248.126' and common_stream_trace_id=1153021139190754263",
"common_client_ip='120.242.132.200' and common_server_ip='116.177.248.126'",
"http_domain='qq.com' or common_server_ip='120.239.72.226'",
"common_server_port not in (80,443)",
"http_domain not like '%qq.com'"
]
},
{
"name":"@index_filter",
"value": [
"common_log_id=1153021139190754263",
"common_client_ip='118.180.48.74'",
"common_client_ip='120.242.132.200'",
"common_server_ip='114.114.114.114'",
"common_server_ip!='114.114.114.114'",
"common_server_ip='120.239.72.226'",
"http_domain='qq.com'",
"http_domain!='qq.com'",
"http_domain='yunser.com'",
"ssl_sni='mmbiz.qpic.cn'",
"ssl_sni='openai.qq.com'",
"common_server_ip like '120.239%'",
"common_server_ip not like '120.239%'",
"common_server_ip like '%114.114%'",
"common_subscriber_id='%test%'",
"http_domain like 'baidu%'",
"http_domain like '%baidu.com%'",
"common_client_ip in ('120.239.72.226','114.114.114.114')",
"common_client_ip not in ('120.239.72.226','114.114.114.114')",
"common_server_ip='116.177.248.126' and notEmpty(http_domain)",
"common_server_ip='116.177.248.126' and common_client_ip='120.242.132.200'",
"common_server_ip='116.177.248.126' and common_stream_trace_id=1153021139190754263",
"common_client_ip='120.242.132.200' and common_server_ip='116.177.248.126'",
"http_domain='qq.com' or common_server_ip='120.239.72.226'"
]
}
]
}

118
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-queries-template.sql Normal file
View File

@@ -0,0 +1,118 @@
--Q01.Count(1)
select count(1) from session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)
--Q02.All Fields Query (default)
SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) LIMIT 30
--Q03.All Fields Query order by Time desc
SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30
--Q04.All Fields Query order by Time asc
SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time asc LIMIT 30
--Q05.All Fields Query by Filter
SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30
--Q06.Default Fields Query by Filter
SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30
--Q07.All Fields Query (sub query by time)
SELECT * FROM session_record AS session_record WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30
--Q08.All Fields Query (sub query by log id)
SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30
--Q09.Default Field Query (sub query by time)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30
--Q10.Default Field Query (sub query by log id)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( select common_log_id FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)) ORDER BY common_recv_time DESC LIMIT 30
--Q11.Default Field Query by Server IP (sub query by log id with Index Table)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_common_server_ip AS session_record_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30
--Q12.Default Field Query by Client IP (sub query by log id with Index Table)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_common_client_ip AS session_record_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30
--Q13.Default Field Query by Domain (sub query by log id with Index Table)
SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_http_domain AS session_record_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30
--Q14.All Fields Query by Client IP (sub query by log id with index Table)
SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_common_client_ip AS session_record_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30
--Q15.All Fields Query by Server IP(sub query by log id with index Table)
SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_common_server_ip AS session_record_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30
--Q16.All Fields Query by Domain(sub query by log id with index Table)
SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_http_domain AS session_record_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30
--Q17.Session Logs Sent to Database Trend(Time Grain 5 minute)
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", count(common_log_id) AS "logs" FROM session_record AS session_record WHERE ( ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ) ) GROUP BY "Receive Time" LIMIT 10000
--Q18.Traffic Bandwidth Trend(Time Grain 30 second)
SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 30 SECOND)))) AS stat_time, sum(common_c2s_byte_num) AS bytes_sent, sum(common_s2c_byte_num) AS bytes_received, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets, sum(common_sessions) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time ORDER BY stat_time ASC LIMIT 10000
--Q19.Log Tend by Type (Time Grain 5 minute)
SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) GROUP BY stat_time, common_schema_type ORDER BY stat_time ASC LIMIT 10000
--Q20.Traffic Metrics Analytic
SELECT round(sum(common_s2c_byte_num) * 8 / 300,2) AS trafficInBits, round(sum(common_c2s_byte_num) * 8 / 300,2) AS trafficOutBits, round(sum(common_s2c_byte_num + common_c2s_byte_num) * 8 / 300,2) AS trafficTotalBits, round(sum(common_s2c_pkt_num) / 300,2) AS trafficInPackets, round(sum(common_c2s_pkt_num) / 300,2) AS trafficOutPackets, round(sum(common_s2c_pkt_num + common_c2s_pkt_num) / 300,2) AS trafficTotalPackets, round(sum(common_sessions) / 300,2) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)
--Q21.Traffic Endpoints Metrics Trend(Time Grain 5 minute)
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", uniq(common_internal_ip) AS "Unique Internal IP", uniq(common_external_ip) AS "Unique External IP", uniq(common_subscriber_id) AS "Unique Subscriber ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000
--Q22.Endpoint Unique Num by L4 Protocol
SELECT 'all' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) UNION ALL SELECT 'tcp' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) UNION ALL SELECT 'UDP' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_UDP', 'IPv6_UDP' )
--Q23.One-sided Connection Trend(Time Grain 5 minute)
SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, (CASE WHEN common_stream_dir = 1 THEN 'c2s' WHEN common_stream_dir = 2 THEN 's2c' WHEN common_stream_dir = 3 THEN 'double' ELSE 'None' END) AS type, sum(common_sessions) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time, common_stream_dir ORDER BY stat_time ASC LIMIT 10000
--Q24. Estimated One-sided Sessions with Bandwidth
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_sessions) AS "sessions", sum(if(common_stream_dir <> 3, common_sessions, 0)) AS "one_side_sessions", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", round(one_side_sessions / sessions, 2) AS one_side_percent FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000
--Q25.Estimated TCP Sequence Gap Loss
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", sum(common_c2s_tcp_lostlen + common_s2c_tcp_lostlen) AS "gap_loss_bytes", round(gap_loss_bytes / bytes, 2) AS gap_loss_percent FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Receive Time" LIMIT 10000
--Q26.Top30 Server IP by Bytes
SELECT "server_ip" AS "server_ip" , SUM(coalesce("bytes",0)) AS "bytes" , SUM(coalesce("bytes_sent",0)) AS "Sent" , SUM(coalesce("bytes_received",0)) AS "Received" , SUM(coalesce("sessions",0)) AS "sessions" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(common_c2s_byte_num+common_s2c_byte_num) AS "bytes" , SUM(coalesce(common_sessions,0)) AS "sessions" , common_server_ip AS "server_ip" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" desc ) GROUP BY "server_ip" ORDER BY "bytes" desc LIMIT 30
--Q27.Top30 Client IP by Sessions
SELECT common_client_ip , COUNT(*) AS sessions FROM session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_client_ip ORDER BY sessions desc LIMIT 0,30
--Q28.Top30 TCP Server Ports by Sessions
SELECT "Server Port" AS "Server Port", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_port AS "Server Port", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Server Port" LIMIT 1048576) GROUP BY "Server Port" ORDER BY "Sessions" DESC LIMIT 30
--Q29.Top30 Domian by Bytes
SELECT "domain" AS "Website Domain" , SUM(coalesce("bytes",0)) AS "Throughput" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "bytes" , http_domain AS "domain" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "domain" ORDER BY "bytes" desc ) GROUP BY "domain" ORDER BY "Throughput" desc LIMIT 30
--Q30.Top30 Endpoint Devices by Bandwidth
SELECT "device_id" AS "device_id", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, common_device_id AS "device_id" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 30
--Q31.Top30 Domain by Unique Client IP
SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Client IP", 0)) AS "Client IP" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Client IP" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 30
--Q32.Top100 Most Time Consuming Domains
SELECT "Domain" AS "Domain", avg(coalesce("Avg Establish Latency(ms)", 0)) AS "Avg Establish Latency(ms)" FROM (SELECT http_domain AS "Domain", avg(coalesce(common_establish_latency_ms, 0)) AS "Avg Establish Latency(ms)" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Avg Establish Latency(ms)" DESC LIMIT 100
--Q33.Top30 Sources by Sessions
SELECT "source" AS "source", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(common_subscriber_id, ''), nullif(common_client_ip, '')) AS "source", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 30
--Q34.Top30 Destinations by Sessions
SELECT "destination" AS "destination", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(http_domain, ''), nullif(common_server_ip, '')) AS "destination", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 30
--Q35.Top30 Destination Regions by Bandwidth
SELECT "server_location" AS "server_location", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT arrayElement(splitByString(',', common_server_location), length(splitByString(',', common_server_location))) AS "server_location", sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "bytes", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 30
--Q36.Top30 URLS by Sessions
SELECT "Http URL" AS "Http URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "Http URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http URL" LIMIT 1048576) GROUP BY "Http URL" ORDER BY "Sessions" DESC LIMIT 30
--Q37.Top30 Destination Transmission APP by Bandwidth
SELECT "server_ip" AS "server_ip", groupUniqArray(coalesce("trans_app", 0)) AS "trans_app", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", groupUniqArray(concat(common_l4_protocol, '/', toString(common_server_port))) AS "trans_app", common_server_ip AS "server_ip" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 30
--Q38.Browsing Users by Website domains and Sessions
SELECT "Subscriber ID" AS "Subscriber ID", "Http.Domain" AS "Http.Domain", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT http_domain AS "Http.Domain", common_subscriber_id AS "Subscriber ID", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) AND notEmpty(common_subscriber_id) ) GROUP BY "Http.Domain", "Subscriber ID" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Subscriber ID", "Http.Domain" ORDER BY "sessions" DESC LIMIT 10000
--Q39.Top Domain and Server IP by Bytes Sent
SELECT "Http.Domain" AS "Http.Domain" , "Server IP" AS "Server IP" , SUM(coalesce("Bytes Sent",0)) AS "Bytes Sent" FROM ( SELECT common_server_ip AS "Server IP" , http_domain AS "Http.Domain" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "Bytes" , SUM(coalesce(common_c2s_byte_num,0)) AS "Bytes Sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "Bytes Received" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "Server IP" , "Http.Domain" ORDER BY "Bytes" desc LIMIT 1048576 ) GROUP BY "Http.Domain" , "Server IP" ORDER BY "Bytes Sent" desc LIMIT 10000
--Q40.Top30 Website Domains by Client IP and Sessions
SELECT "Http.Domain" AS "Http.Domain", "Client IP" AS "Client IP", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT common_client_ip AS "Client IP", http_domain AS "Http.Domain", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Client IP", "Http.Domain" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Http.Domain", "Client IP" ORDER BY "sessions" DESC LIMIT 10000
--Q41.Domain is Accessed by Unique Client IP Trend(bytes Time Grain 5 minute)
SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , http_domain AS Domain, COUNT(DISTINCT(common_client_ip)) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) AND http_domain IN ( SELECT http_domain FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY SUM(common_s2c_byte_num+common_c2s_byte_num) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , http_domain ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000
--Q42. Domain is Accessed by Unique Client IP Trend(sessions,Time Grain 5 minute)
SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),3600)*3600) AS stat_time , http_domain , uniq (common_client_ip) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start)-604800 AND common_recv_time < toDateTime(@end) AND http_domain IN ( SELECT http_domain FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY COUNT(*) desc LIMIT 5 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 3600)*3600), http_domain ORDER BY stat_time desc LIMIT 10000
--Q43.Bandwidth Trend with Device ID(Time Grain 5 minute)
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", common_device_id AS "Device ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time", "Device ID" LIMIT 10000
--Q44.Internal IP by Sled IP and Sessions
SELECT "Internal IP" AS "Internal IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_internal_ip AS "Internal IP", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Sled IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 10000
--Q45.Bandwidth Trend with Internal IP (Time Grain 5 minute)
SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_c2s_pkt_num + common_s2c_pkt_num, 0)) AS "Packets", sum(coalesce(common_sessions, 0)) AS "New Sessions", sum(coalesce(common_c2s_byte_num, 0)) AS "Bytes Sent", sum(coalesce(common_s2c_byte_num, 0)) AS "Bytes Received", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent", sum(coalesce(common_s2c_pkt_num, 0)) AS "Packets Received" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) GROUP BY "Receive Time" LIMIT 10000
--Q46.Top30 Domains Detail with Internal IP
SELECT "Domain" AS "Domain", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_domain AS "Domain", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Sessions" DESC LIMIT 30
--Q47.Top30 URLS Detail with Internal IP
SELECT "URL" AS "URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_url) ) GROUP BY "URL" LIMIT 1048576) GROUP BY "URL" ORDER BY "Sessions" DESC LIMIT 30
--Q48.Top Domains with Unique Client IP and Subscriber ID
SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Unique Client IP", 0)) AS "Unique Client IP", sum(coalesce("Unique Subscriber ID", 0)) AS "Unique Subscriber ID" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Unique Client IP", uniq(common_subscriber_id) AS "Unique Subscriber ID" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Unique Client IP" DESC LIMIT 100
--Q49.Top100 Domains by Packets sent
SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Packets Sent", 0)) AS "Packets Sent" FROM (SELECT http_domain AS "Http.Domain", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Packets Sent" DESC LIMIT 100
--Q50.Internal and External asymmetric traffic
SELECT "Internal IP" AS "Internal IP", "External IP" AS "External IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_external_ip AS "External IP", common_internal_ip AS "Internal IP", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes Sent+Bytes Received", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Sled IP", "External IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "External IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 500
--Q51.Client and Server ASN asymmetric traffic
SELECT "Client ASN" AS "Client ASN", "Server ASN" AS "Server ASN", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_asn AS "Server ASN", common_client_asn AS "Client ASN", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Server ASN", "Client ASN" LIMIT 1048576) GROUP BY "Client ASN", "Server ASN" ORDER BY "Sessions" DESC LIMIT 500
--Q52.Top handshake latency by Website and Client IPs
SELECT "SSL.SNI" AS "SSL.SNI", "Client IP" AS "Client IP", avg(coalesce("Establish Latency(ms)", 0)) AS "Establish Latency(ms)" FROM (SELECT common_client_ip AS "Client IP", ssl_sni AS "SSL.SNI", avg(coalesce(common_establish_latency_ms, 0)) AS "Establish Latency(ms)" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Client IP", "SSL.SNI" LIMIT 1048576) GROUP BY "SSL.SNI", "Client IP" ORDER BY "Establish Latency(ms)" DESC LIMIT 500
--Q53.Domain baidu.com Drill down Client IP
select common_client_ip as "Client IP" , avg(common_establish_latency_ms) as "Establishing Time Mean(ms)", count(1) as Responses,any(common_client_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Client IP" order by Responses desc limit 100
--Q54.Domain baidu.com Drill down Server IP
select common_server_ip as "Server IP" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses,any(common_server_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Server IP" order by Responses desc limit 100
--Q55.Domain baidu.com Drill down URI
select http_url as "URI" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "URI" order by Responses desc limit 100
--Q56.L7 Protocol Metrics
select common_l7_protocol as "Protocol" , uniq(common_client_ip) as "Clients" , uniq(common_server_ip) as "Servers", count(1) as Sessions,sum(common_c2s_byte_num+common_s2c_byte_num) as bytes from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and notEmpty(common_l7_protocol) group by common_l7_protocol order by bytes desc
--Q57.L7 Protocol SIP Drill down Client IP
select common_client_ip as "Client IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_client_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Client IP" order by Sessions desc limit 100
--Q58.L7 Protocol SIP Drill down Server IP
select common_server_ip as "Server IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_server_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Server IP" order by Sessions desc limit 100
--Q59.Top5 Server IP keys with Unique Client IPs Trend (Grain 5 minute)
SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , common_server_ip AS server_ip, COUNT(DISTINCT(common_client_ip)) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_server_ip IN ( SELECT common_server_ip FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_server_ip ORDER BY count(*) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , server_ip ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000

11
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/columns_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "columns_cluster",
"fields": [
{
"name": "database",
"type": "string"
}
]
}

11
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/disks_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "disks_cluster",
"fields": [
{
"name": "name",
"type": "string"
}
]
}

6
TSG发布版本更新记录/TSG-21.12/qgw/schema/dos_event.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/dos_event.json
View File

@@ -187,9 +187,9 @@
"session_rate"
],
"internal_columns": [
"common_recv_time",
"common_log_id",
"common_processing_time"
"start_time",
"log_id",
"end_time"
]
},
"fields": [

21
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-filter.json Normal file
View File

@@ -0,0 +1,21 @@
{
"version": "1.0",
"name": "druid-Raw",
"namespace": "druid",
"filters": [
{
"name":"@start",
"value": "'2021-10-19 10:00:00'"
},
{
"name":"@end",
"value": "'2021-10-20 11:00:00'"
},
{
"name":"@common_filter",
"value": [
"common_client_ip='192.168.44.21'and common_server_port=443"
]
}
]
}

92
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-queries-template.sql Normal file
View File

@@ -0,0 +1,92 @@
--Q01.All Security Event Hits
select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end group by policy_id
--Q02.Security Event Hits with Policy ID 0
select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id in (0) group by policy_id
--Q03.All Security Event Hits Trend by 5min A
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from security_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 10000
--Q04.Security Event Hit Time（first and last time) A
select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from security_event_hits_log where policy_id in (0) group by policy_id
--Q05.Top 200 Security Policies
select policy_id, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id order by hits desc limit 200
--Q06.Top 200 Security Policies with Action
select policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=@start and __time <@end group by policy_id, action order by hits desc limit 200
--Q07.All Proxy Event Hits
select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id
--Q08.Proxy Event Hits with Policy ID 0
select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=0 group by policy_id
--Q09.All Proxy Event Hits Trend by 5min A
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 10000
--Q10.Proxy Event Hit Time（first and last time) A
select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from proxy_event_hits_log where policy_id in (0) group by policy_id
--Q11.Top 200 Proxy Policies
select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id order by hits desc limit 200
--Q12.Top 200 Proxy Policies with sub Action
select policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id, sub_action order by hits desc limit 200
--Q13.Proxy Action Hits
select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by sub_action
--Q14.Proxy Action Hits Trend by 5min
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') , sub_action limit 10000
--Q15.Traffic Metrics Pinning Hits
SELECT sum(not_pinning_num) AS sessions, 'notPinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(pinning_num) AS sessions, 'pinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(maybe_pinning_num) AS sessions, 'maybePinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end
--Q16.Traffic Metrics Pinning Trend by 5Min
SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000
--Q17.Traffic Metrics Not Pinning Trend by 5Min
SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(not_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time>= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000
--Q18.Traffic Metrics Maybe Pinning Trend by 5Min
SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(maybe_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000
--Q19.Traffic Metrics Throughput Bytes IN/OUT
select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q20. Traffic Metrics Throughput Packets IN/OUT
select sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q21.Traffic Metrics New Sessions
select sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q22.Traffic Metrics Bandwidth Bytes IN/OUT
select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')
--Q23.Traffic Metrics Bandwidth Packets IN/OUT
select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')
--Q24.Traffic Metrics New Sessions Trend by 5Min
select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')
--Q25.Traffic Metrics New and Live Sessions
select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q26.Traffic Metrics New and Live Sessions Trend by 5Min
select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')
--Q27.Traffic Metrics Security Throughput Bytes
select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end
--Q28.Traffic Metrics Security Throughput Packets
select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q29.Traffic Metrics Security Throughput Sessions
select sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end
--Q30.Traffic Metrics Security Bandwidth Bytes by 5Min
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_bytes' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_bytes' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_bytes' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_bytes' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_bytes' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')
--Q31.Traffic Metrics Security Bandwidth Packets by 5Min
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_packets' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_packets' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_packets' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_packets' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')
--Q32.Traffic Metrics Security Sessions Trend by 5Min
select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_conn_num' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_conn_num' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_conn_num' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_conn_num' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_conn_num' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')
--Q33.Top 100 Client IP by Sessions
select source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_client_ip_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by source order by sessions desc limit 100
--Q34.Top 100 Server IP by Sessions
select destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_server_ip_log where __time >= @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100
--Q35.Top 100 Internal IP by Sessions
select source as internal_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by source order by sessions desc limit 100
--Q36.Top 100 External IP by Sessions
select destination as external_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >= @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100
--Q37.Top 100 Domain by Bytes
select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='bytes' group by domain order by bytes desc limit 100
--Q38.Top 100 Subscriber ID by Sessions
select subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by subscriber_id order by sessions desc limit 100
--Q39.Top 100 Hit URLS by hits
select url,sum(session_num) as hits from top_urls_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by url order by hits desc limit 100
--Q40.Proxy Event Unique ISP
SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end group by policy_id
--Q41.Traffic Composition Metrics
SELECT APPROX_COUNT_DISTINCT_DS_HLL(ip_object) AS uniq_client_ip, SUM(one_sided_connections) AS one_sided_connections, SUM(uncategorized_bytes) AS total_uncategorized_bytes, SUM(fragmentation_packets) AS fragmentation_packets, SUM(sequence_gap_loss) AS sequence_gap_loss_bytes, SUM(s2c_byte_num+c2s_byte_num) AS summaryTotalBytes, SUM(s2c_pkt_num+c2s_pkt_num) AS summaryTotalPackets, SUM(sessions) AS summarySessions FROM traffic_summary_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end LIMIT 1
--Q42.Traffic Composition Throughput
(SELECT SUM(c2s_byte_num + s2c_byte_num) as total_bytes, SUM(sessions) as total_sessions, (SUM(c2s_byte_num + s2c_byte_num) * 8)/((TIMESTAMP_TO_MILLIS(TIMESTAMP @end )-TIMESTAMP_TO_MILLIS(TIMESTAMP @start ))/1000) AS data_rate FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' LIMIT 1) UNION ALL ( SELECT SUM(sessions), 0, 0 FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' GROUP BY __time ORDER BY __time DESC LIMIT 1 )
--Q43.Traffic Composition Protocol Tree
SELECT protocol_id, SUM(sessions) as sessions,SUM(c2s_byte_num) as c2s_byte_num, SUM(c2s_pkt_num) as c2s_pkt_num, SUM(s2c_byte_num) as s2c_byte_num, SUM(s2c_pkt_num) as s2c_pkt_num FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end GROUP BY protocol_id
--Q44.System Quota
SELECT log_type, SUM(used_size) as used_size, SUM(max_size) * 7/10 as max_size, TIME_FORMAT(LATEST(last_storage) * 1000,'YYYY-MM-dd') as first_storage FROM ( SELECT log_type, LATEST(used_size) as used_size, LATEST(max_size) as max_size, LATEST(last_storage) as last_storage FROM sys_storage_log WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '1' HOUR AND data_center != '' GROUP BY data_center,log_type ) GROUP BY log_type
--Q45.System Quota Daily Trend
select TIME_FORMAT(__time,'YYYY-MM-dd') as stat_time,log_type as type, sum(aggregate_size) as used_size from sys_storage_log where __time >= @start and __time < @end group by TIME_FORMAT(__time,'YYYY-MM-dd'), log_type
--Q46.Traffic Statistics(Metrics01)
select sum(total_hit_sessions) as total_hit_sessions, sum(total_bytes_transferred) as total_bytes_transferred, sum(total_packets_transferred) as total_packets_transferred, sum(total_new_sessions) as total_new_sessions , sum(total_close_sessions) as total_close_sessions, sum(average_new_sessions_per_second) as average_new_sessions_per_second , sum(average_bytes_per_second) as average_bytes_per_second , sum(average_packets_per_second) as average_packets_per_second , COUNT(DISTINCT(device_id)) as device_num, sum(live_sessions) as average_live_sessions from ( select device_id, sum(intercept_conn_num + monitor_conn_num + deny_conn_num + allow_conn_num) as total_hit_sessions, sum(total_in_bytes + total_out_bytes) as total_bytes_transferred, sum(total_in_packets + total_out_packets) as total_packets_transferred, sum(new_conn_num) as total_new_sessions, sum(close_conn_num) as total_close_sessions, avg(nullif(new_conn_num, 0))/ 5 as average_new_sessions_per_second, avg(nullif(total_in_bytes + total_out_bytes, 0))* 8 / 5 as average_bytes_per_second, avg(nullif(total_in_packets + total_out_packets, 0))/ 5 as average_packets_per_second, avg(nullif(established_conn_num, 0)) as live_sessions from traffic_metrics_log where __time >= @start and __time < @end group by device_id)

53
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/engine-filter.json Normal file
View File

@@ -0,0 +1,53 @@
{
"version": "1.0",
"name": "Engine-Raw",
"namespace": "Engine",
"filters": [
{
"name":"@start",
"value": "'2021-10-19 10:00:00'"
},
{
"name":"@end",
"value": "'2021-10-20 11:00:00'"
},
{
"name":"@common_filter",
"value": [
"common_log_id=1153021139190754263",
"common_client_ip='36.189.226.21'",
"common_internal_ip='223.116.37.192'",
"common_server_ip='8.8.8.8'",
"common_external_ip='111.10.53.14'",
"common_client_port=52607",
"common_server_port=443",
"common_c2s_pkt_num>5",
"common_s2c_pkt_num>5",
"common_c2s_byte_num>100",
"common_s2c_byte_num<200",
"common_schema_type='DNS'",
"common_establish_latency_ms>200",
"common_con_duration_ms>10000",
"common_stream_trace_id=1153021139190754263",
"common_tcp_client_isn=2857077935",
"common_tcp_server_isn=0",
"http_domain='microsoft.com'",
"mail_account='abc@xx.com'",
"mail_subject='test'",
"dns_qname='qbwup.imtt.qq.com'",
"ssl_sni='note.youdao.com'",
"ssl_con_latency_ms>100",
"ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'",
"common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'",
"common_server_ip='111.10.53.14' and common_server_port=443",
"mail_account like 'abc@%'",
"http_domain like '%baidu.com%'",
"ssl_sni like '%youdao.com'",
"common_client_ip in ('36.189.226.21','111.10.53.14')",
"common_server_port not in (80,443)",
"notEmpty(http_domain)",
"http_domain not like '%microsoft.com'"
]
}
]
}

0
TSG发布版本更新记录/TSG-21.12/qgw/schema/engine-queries-template.sql → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/engine-queries-template.sql
View File

2
TSG发布版本更新记录/TSG-21.12/qgw/schema/gtpc_record.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/gtpc_record.json
View File

@@ -780,7 +780,7 @@
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
},
"allow_query": "true"
"visibility": "hidden"
},
"type": "string"
},

0
TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-filter.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/hbase-filter.json
View File

0
TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-queries-template.sql → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/hbase-queries-template.sql
View File

6
TSG发布版本更新记录/TSG-21.12/qgw/schema/interim_session_record.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/interim_session_record.json
View File

@@ -263,7 +263,9 @@
"internal_columns": [
"common_recv_time",
"common_log_id",
"common_processing_time"
"common_processing_time",
"common_userdefine_app_name",
"common_tunnels"
],
"tunnel_type": {
"$ref": "public_schema_info.json#/tunnel_type"
@@ -747,7 +749,7 @@
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
},
"allow_query": "true"
"visibility": "hidden"
},
"type": "string"
},

7
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/job_result.json Normal file
View File

@@ -0,0 +1,7 @@
{
"type": "record",
"name": "job_result",
"namespace": "tsg_galaxy",
"fields": [
]
}

169
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_interim.json Normal file
View File

@@ -0,0 +1,169 @@
{
"type": "record",
"name": "liveChart_interim",
"in": "INTERIM-SESSION-RECORD",
"out": "TRAFFIC-PROTOCOL-STAT",
"task": "Protocol-Distribution",
"doc": {
"timestamp": {
"name": "stat_time",
"type": "long"
},
"dimensions": [
{
"name": "protocol_id",
"fieldName": "common_protocol_label",
"type": "string"
},
{
"name": "entrance_id",
"fieldName": "common_entrance_id",
"type": "string"
},
{
"name": "isp",
"fieldName": "common_isp",
"type": "string"
},
{
"name": "data_center",
"fieldName": "common_data_center",
"type": "string"
},
{
"name": "device_group",
"fieldName": "common_device_group",
"type": "string"
}
],
"metrics": [
{
"function": "sum",
"name": "sessions",
"fieldName": "common_sessions",
"type": "long"
},
{
"function": "sum",
"name": "c2s_byte_num",
"fieldName": "common_c2s_byte_diff",
"type": "long"
},
{
"function": "sum",
"name": "s2c_byte_num",
"fieldName": "common_s2c_byte_diff",
"type": "long"
},
{
"function": "sum",
"name": "c2s_pkt_num",
"fieldName": "common_c2s_pkt_diff",
"type": "long"
},
{
"function": "sum",
"name": "s2c_pkt_num",
"fieldName": "common_s2c_pkt_diff",
"type": "long"
},
{
"function": "sum",
"name": "c2s_ipfrag_num",
"fieldName": "common_c2s_ipfrag_num",
"type": "long"
},
{
"function": "sum",
"name": "s2c_ipfrag_num",
"fieldName": "common_s2c_ipfrag_num",
"type": "long"
},
{
"function": "sum",
"name": "c2s_tcp_lostlen",
"fieldName": "common_c2s_tcp_lostlen",
"type": "long"
},
{
"function": "sum",
"name": "s2c_tcp_lostlen",
"fieldName": "common_s2c_tcp_lostlen",
"type": "long"
},
{
"function": "sum",
"name": "c2s_tcp_unorder_num",
"fieldName": "common_c2s_tcp_unorder_num",
"type": "long"
},
{
"function": "sum",
"name": "s2c_tcp_unorder_num",
"fieldName": "common_s2c_tcp_unorder_num",
"type": "long"
},
{
"function": "disCount",
"name": "unique_sip_num",
"fieldName": "common_server_ip",
"type": "long"
},
{
"function": "disCount",
"name": "unique_cip_num",
"fieldName": "common_client_ip",
"type": "long"
}
],
"filters": [
{
"fieldName": "common_protocol_label",
"type": "notempty"
}
],
"transforms": [
{
"function": "combination",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "common_l7_protocol,."
},
{
"function": "combination",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "common_app_label,."
},
{
"function": "flattenSpec",
"name": "data_center",
"fieldName": "common_device_tag",
"parameters": "$.tags[?(@.tag=='data_center')].value"
},
{
"function": "flattenSpec",
"name": "device_group",
"fieldName": "common_device_tag",
"parameters": "$.tags[?(@.tag=='device_group')].value"
},
{
"function": "hierarchy",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "."
}
],
"action": [
{
"label": "Default",
"metrics": "c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num"
}
],
"granularity": {
"type": "period",
"period": "15S"
}
},
"fields": []
}

169
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_session.json Normal file
View File

@@ -0,0 +1,169 @@
{
"type": "record",
"name": "liveChart_session",
"in": "SESSION-RECORD",
"out": "TRAFFIC-PROTOCOL-STAT",
"task": "Protocol-Distribution",
"doc": {
"timestamp": {
"name": "stat_time",
"type": "long"
},
"dimensions": [
{
"name": "protocol_id",
"fieldName": "common_protocol_label",
"type": "string"
},
{
"name": "entrance_id",
"fieldName": "common_entrance_id",
"type": "string"
},
{
"name": "isp",
"fieldName": "common_isp",
"type": "string"
},
{
"name": "data_center",
"fieldName": "common_data_center",
"type": "string"
},
{
"name": "device_group",
"fieldName": "common_device_group",
"type": "string"
}
],
"metrics": [
{
"function": "sum",
"name": "sessions",
"fieldName": "common_sessions",
"type": "long"
},
{
"function": "sum",
"name": "c2s_byte_num",
"fieldName": "common_c2s_byte_diff",
"type": "long"
},
{
"function": "sum",
"name": "s2c_byte_num",
"fieldName": "common_s2c_byte_diff",
"type": "long"
},
{
"function": "sum",
"name": "c2s_pkt_num",
"fieldName": "common_c2s_pkt_diff",
"type": "long"
},
{
"function": "sum",
"name": "s2c_pkt_num",
"fieldName": "common_s2c_pkt_diff",
"type": "long"
},
{
"function": "sum",
"name": "c2s_ipfrag_num",
"fieldName": "common_c2s_ipfrag_num",
"type": "long"
},
{
"function": "sum",
"name": "s2c_ipfrag_num",
"fieldName": "common_s2c_ipfrag_num",
"type": "long"
},
{
"function": "sum",
"name": "c2s_tcp_lostlen",
"fieldName": "common_c2s_tcp_lostlen",
"type": "long"
},
{
"function": "sum",
"name": "s2c_tcp_lostlen",
"fieldName": "common_s2c_tcp_lostlen",
"type": "long"
},
{
"function": "sum",
"name": "c2s_tcp_unorder_num",
"fieldName": "common_c2s_tcp_unorder_num",
"type": "long"
},
{
"function": "sum",
"name": "s2c_tcp_unorder_num",
"fieldName": "common_s2c_tcp_unorder_num",
"type": "long"
},
{
"function": "disCount",
"name": "unique_sip_num",
"fieldName": "common_server_ip",
"type": "long"
},
{
"function": "disCount",
"name": "unique_cip_num",
"fieldName": "common_client_ip",
"type": "long"
}
],
"filters": [
{
"fieldName": "common_protocol_label",
"type": "notempty"
}
],
"transforms": [
{
"function": "combination",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "common_l7_protocol,."
},
{
"function": "combination",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "common_app_label,."
},
{
"function": "flattenSpec",
"name": "data_center",
"fieldName": "common_device_tag",
"parameters": "$.tags[?(@.tag=='data_center')].value"
},
{
"function": "flattenSpec",
"name": "device_group",
"fieldName": "common_device_tag",
"parameters": "$.tags[?(@.tag=='device_group')].value"
},
{
"function": "hierarchy",
"name": "protocol_id",
"fieldName": "common_protocol_label",
"parameters": "."
}
],
"action": [
{
"label": "Default",
"metrics": "sessions,c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num,c2s_ipfrag_num,s2c_ipfrag_num,c2s_tcp_lostlen,s2c_tcp_lostlen,c2s_tcp_unorder_num,s2c_tcp_unorder_num"
}
],
"granularity": {
"type": "period",
"period": "15S"
}
},
"fields": []
}

4
TSG发布版本更新记录/TSG-21.12/qgw/schema/meta_data.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/meta_data.json
View File

@@ -81,8 +81,8 @@
"group": "HBASE_GROUP",
"tables": [
"relation_account_framedip",
"job_result",
"recommendation_app_cip"
"recommendation_app_cip",
"job_result"
]
}
]

11
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/parts_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "parts_cluster",
"fields": [
{
"name": "name",
"type": "string"
}
]
}

11
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/processes.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "processes",
"fields": [
{
"name": "query_id",
"type": "string"
}
]
}

6
TSG发布版本更新记录/TSG-21.12/qgw/schema/proxy_event.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event.json
View File

@@ -633,6 +633,10 @@
{
"code": "hijack",
"value": "Hijack"
},
{
"code": "edit_element",
"value": "Edit Element"
}
],
"allow_query": "true"
@@ -701,7 +705,7 @@
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
},
"allow_query": "true"
"visibility": "hidden"
},
"type": "string"
},

58
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event_hits_log.json Normal file
View File

@@ -0,0 +1,58 @@
{
"type": "record",
"name": "proxy_event_hits_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "action",
"type": "long"
},
{
"name": "isp",
"type": "string"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "hits",
"type": "long"
},
{
"name": "policy_id",
"type": "long"
},
{
"name": "sub_action",
"type": "string"
},
{
"name": "country",
"type": "string"
},
{
"name": "location",
"type": "string"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "ip_object",
"type": "string"
}
]
}

2
TSG发布版本更新记录/TSG-21.12/qgw/schema/public_schema_info.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/public_schema_info.json
View File

@@ -1973,4 +1973,4 @@
]
}
}
}
}

11
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "query_log",
"fields": [
{
"name": "query_id",
"type": "string"
}
]
}

11
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "query_log_cluster",
"fields": [
{
"name": "type",
"type": "string"
}
]
}

37
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_onff_log.json Normal file
View File

@@ -0,0 +1,37 @@
{
"type": "record",
"name": "radius_onff_log",
"namespace": "tsg_galaxy_v3",
"fields": [
{
"name": "event_timestamp",
"label": "Event Time",
"type": "long"
},
{
"name": "account",
"label": "Account",
"type": "string"
},
{
"name": "framed_ip",
"label": "Framed IP",
"type": "string"
},
{
"name": "acct_session_id",
"label": "Acct Session ID",
"type": "string"
},
{
"name": "acct_status_type",
"label": "Acct Status Type",
"type": "int"
},
{
"name": "acct_session_time",
"label": "Acct Session Time",
"type": "int"
}
]
}

2
TSG发布版本更新记录/TSG-21.12/qgw/schema/radius_record.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_record.json
View File

@@ -615,7 +615,7 @@
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
},
"allow_query": "true"
"visibility": "hidden"
},
"type": "string"
},

0
TSG发布版本更新记录/TSG-21.12/qgw/schema/recommendation_app_cip.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/recommendation_app_cip.json
View File

7
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/relation_account_framedip.json Normal file
View File

@@ -0,0 +1,7 @@
{
"type": "record",
"name": "relation_account_framedip",
"namespace": "tsg_galaxy",
"fields": [
]
}

7
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/report_result.json Normal file
View File

@@ -0,0 +1,7 @@
{
"type": "record",
"name": "report_result",
"namespace": "tsg",
"fields": [
]
}

6
TSG发布版本更新记录/TSG-21.12/qgw/schema/security_event.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event.json
View File

@@ -307,7 +307,9 @@
"internal_columns": [
"common_recv_time",
"common_log_id",
"common_processing_time"
"common_processing_time",
"common_userdefine_app_name",
"common_tunnels"
],
"tunnel_type": {
"$ref": "public_schema_info.json#/tunnel_type"
@@ -809,7 +811,7 @@
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
},
"allow_query": "true"
"visibility": "hidden"
},
"type": "string"
},

42
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event_hits_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "security_event_hits_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "action",
"type": "long"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "hits",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "isp",
"type": "string"
},
{
"name": "policy_id",
"type": "long"
}
]
}

6
TSG发布版本更新记录/TSG-21.12/qgw/schema/session_record.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record.json
View File

@@ -282,7 +282,9 @@
"internal_columns": [
"common_recv_time",
"common_log_id",
"common_processing_time"
"common_processing_time",
"common_userdefine_app_name",
"common_tunnels"
],
"tunnel_type": {
"$ref": "public_schema_info.json#/tunnel_type"
@@ -778,7 +780,7 @@
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
},
"allow_query": "true"
"visibility": "hidden"
},
"type": "string"
},

71
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_client_ip.json Normal file
View File

@@ -0,0 +1,71 @@
{
"type": "record",
"name": "session_record_common_client_ip",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"index_key": "common_client_ip"
},
"fields": [
{
"name": "common_log_id",
"type": "long"
},
{
"name": "common_recv_time",
"type": "long"
},
{
"name": "common_server_ip",
"type": "string"
},
{
"name": "common_client_ip",
"type": "string"
},
{
"name": "common_sled_ip",
"type": "string"
},
{
"name": "common_entrance_id",
"type": "int"
},
{
"name": "common_subscriber_id",
"type": "string"
},
{
"name": "common_stream_trace_id",
"type": "long"
},
{
"name": "common_schema_type",
"type": "string"
},
{
"name": "common_client_port",
"type": "int"
},
{
"name": "common_server_port",
"type": "int"
},
{
"name": "common_app_label",
"type": "string"
},
{
"name": "common_direction",
"type": "int"
},
{
"name": "http_domain",
"type": "string"
},
{
"name": "ssl_sni",
"type": "string"
}
]
}

71
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_server_ip.json Normal file
View File

@@ -0,0 +1,71 @@
{
"type": "record",
"name": "session_record_common_server_ip",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"index_key": "common_server_ip"
},
"fields": [
{
"name": "common_log_id",
"type": "long"
},
{
"name": "common_recv_time",
"type": "long"
},
{
"name": "common_server_ip",
"type": "string"
},
{
"name": "common_client_ip",
"type": "string"
},
{
"name": "common_sled_ip",
"type": "string"
},
{
"name": "common_entrance_id",
"type": "int"
},
{
"name": "common_subscriber_id",
"type": "string"
},
{
"name": "common_stream_trace_id",
"type": "long"
},
{
"name": "common_schema_type",
"type": "string"
},
{
"name": "common_client_port",
"type": "int"
},
{
"name": "common_server_port",
"type": "int"
},
{
"name": "common_app_label",
"type": "string"
},
{
"name": "common_direction",
"type": "int"
},
{
"name": "http_domain",
"type": "string"
},
{
"name": "ssl_sni",
"type": "string"
}
]
}

71
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_http_domain.json Normal file
View File

@@ -0,0 +1,71 @@
{
"type": "record",
"name": "session_record_http_domain",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"index_key": "http_domain"
},
"fields": [
{
"name": "common_log_id",
"type": "long"
},
{
"name": "common_recv_time",
"type": "long"
},
{
"name": "common_server_ip",
"type": "string"
},
{
"name": "common_client_ip",
"type": "string"
},
{
"name": "common_sled_ip",
"type": "string"
},
{
"name": "common_entrance_id",
"type": "int"
},
{
"name": "common_subscriber_id",
"type": "string"
},
{
"name": "common_stream_trace_id",
"type": "long"
},
{
"name": "common_schema_type",
"type": "string"
},
{
"name": "common_client_port",
"type": "int"
},
{
"name": "common_server_port",
"type": "int"
},
{
"name": "common_app_label",
"type": "string"
},
{
"name": "common_direction",
"type": "int"
},
{
"name": "http_domain",
"type": "string"
},
{
"name": "ssl_sni",
"type": "string"
}
]
}

801
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_packet_capture_event.json Normal file
View File

@@ -0,0 +1,801 @@
{
"type": "record",
"name": "sys_packet_capture_event",
"namespace": "tsg_galaxy_v3",
"doc": {
"primary_key": "common_log_id",
"partition_key": "common_recv_time"
},
"fields": [
{
"name": "common_recv_time",
"type": "long",
"doc": {
"allow_query": "true",
"constraints": {
"type": "timestamp"
},
"format": {
"functions": "current_timestamp"
}
},
"label": "Receive Time"
},
{
"name": "common_log_id",
"type": "long",
"doc": {
"allow_query": "true",
"format": {
"functions": "snowflake_id"
}
},
"label": "Log ID"
},
{
"name": "common_policy_id",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Policy ID"
},
{
"name": "common_subscriber_id",
"type": "string",
"doc": {
"allow_query": "true"
},
"label": "Subscriber ID"
},
{
"name": "common_imei",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "IMEI"
},
{
"name": "common_imsi",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "IMSI"
},
{
"name": "common_phone_number",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "Phone Number"
},
{
"name": "common_client_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "Client IP"
},
{
"name": "common_internal_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "Internal IP"
},
{
"name": "common_client_port",
"type": "int",
"label": "Client Port"
},
{
"name": "common_l4_protocol",
"type": "string",
"label": "L4 Protocol"
},
{
"name": "common_address_type",
"type": "int",
"doc": {
"data": [
{
"code": "4",
"value": "ipv4"
},
{
"code": "6",
"value": "ipv6"
}
]
},
"label": "Address Type"
},
{
"name": "common_server_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "Server IP"
},
{
"name": "common_server_port",
"type": "int",
"doc": {
"allow_query": "true"
},
"label": "Server Port"
},
{
"name": "common_external_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "External IP"
},
{
"name": "common_action",
"type": "int",
"doc": {
"allow_query": "true",
"data": [
{
"code": "0",
"value": "None"
},
{
"code": "1",
"value": "Monitor"
},
{
"code": "2",
"value": "Intercept"
},
{
"code": "16",
"value": "Deny"
},
{
"code": "128",
"value": "Allow"
}
]
},
"label": "Action"
},
{
"name": "common_direction",
"type": "int",
"doc": {
"data": [
{
"code": "69",
"value": "outbound"
},
{
"code": "73",
"value": "inbound"
}
]
},
"label": "Direction"
},
{
"name": "common_entrance_id",
"type": "int",
"doc": {
"visibility": "disabled"
},
"label": "Entrance ID"
},
{
"name": "common_sled_ip",
"type": "string",
"doc": {
"allow_query": "true",
"constraints": {
"type": "ip"
}
},
"label": "Sled IP"
},
{
"name": "common_client_location",
"type": "string",
"label": "Client Location"
},
{
"name": "common_client_asn",
"type": "string",
"label": "Client ASN"
},
{
"name": "common_server_location",
"type": "string",
"label": "Server Location"
},
{
"name": "common_server_asn",
"type": "string",
"label": "Server ASN"
},
{
"name": "common_sessions",
"type": "long",
"label": "Sessions"
},
{
"name": "common_c2s_pkt_num",
"type": "long",
"label": "Packets Sent"
},
{
"name": "common_s2c_pkt_num",
"type": "long",
"label": "Packets Received"
},
{
"name": "common_c2s_byte_num",
"type": "long",
"label": "Bytes Sent"
},
{
"name": "common_s2c_byte_num",
"type": "long",
"label": "Bytes Received"
},
{
"name": "common_c2s_pkt_diff",
"label": "Packets Sent(Diff)",
"type": "long"
},
{
"name": "common_s2c_pkt_diff",
"label": "Packets Received(Diff)",
"type": "long"
},
{
"name": "common_c2s_byte_diff",
"label": "Bytes Sent(Diff)",
"type": "long"
},
{
"name": "common_s2c_byte_diff",
"label": "Bytes Received(Diff)",
"type": "long"
},
{
"name": "common_service",
"type": "int",
"doc": {
"visibility": "disabled"
},
"label": "Service"
},
{
"name": "common_schema_type",
"type": "string",
"doc": {
"data": [
{
"code": "BASE",
"value": "BASE"
},
{
"code": "HTTP",
"value": "HTTP"
},
{
"code": "MAIL",
"value": "MAIL"
},
{
"code": "DNS",
"value": "DNS"
},
{
"code": "SSL",
"value": "SSL"
},
{
"code": "FTP",
"value": "FTP"
}
],
"visibility": "hidden"
},
"label": "Schema Type"
},
{
"name": "common_user_tags",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "User Tags"
},
{
"name": "common_sub_action",
"type": "string",
"doc": {
"data": [
{
"code": "allow",
"value": "Allow"
},
{
"code": "deny",
"value": "Deny"
},
{
"code": "monitor",
"value": "Monitor"
},
{
"code": "replace",
"value": "Replace"
},
{
"code": "redirect",
"value": "Redirect"
},
{
"code": "insert",
"value": "Insert"
},
{
"code": "hijack",
"value": "Hijack"
}
],
"visibility": "hidden"
},
"label": "Sub Action"
},
{
"name": "common_user_region",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "User Region"
},
{
"name": "common_device_id",
"type": "string",
"label": "Device ID"
},
{
"name": "common_egress_link_id",
"label": "Egress Link ID",
"doc": {
"visibility": "hidden"
},
"type": "int"
},
{
"name": "common_ingress_link_id",
"label": "Ingress Link ID",
"doc": {
"visibility": "hidden"
},
"type": "int"
},
{
"name": "common_isp",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "ISP"
},
{
"name": "common_device_tag",
"type": "string",
"doc": {
"visibility": "hidden",
"format": {
"functions": "flattenSpec,flattenSpec",
"appendTo": "common_data_center,common_device_group",
"param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value"
}
},
"label": "Device Tag"
},
{
"name": "common_data_center",
"label": "Data Center",
"doc": {
"constraints": {
"operator_functions": "=,!="
},
"data": {
"$ref": "device_tag.json#",
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
}
},
"type": "string"
},
{
"name": "common_device_group",
"label": "Device Group",
"doc": {
"constraints": {
"operator_functions": "=,!="
},
"data": {
"$ref": "device_tag.json#",
"key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']",
"value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']"
}
},
"type": "string"
},
{
"name": "common_encapsulation",
"type": "int",
"doc": {
"data": [
{
"code": "0",
"value": "Ethernet"
},
{
"code": "8",
"value": "PPP"
},
{
"code": "12",
"value": "CiscoHDLC"
}
]
},
"label": "Encapsulation"
},
{
"name": "common_app_label",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "Application Label"
},
{
"name": "common_tunnels",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Tunnels"
},
{
"name": "common_protocol_label",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Protocol Label"
},
{
"name": "common_app_id",
"type": "string",
"label": "Application ID",
"doc": {
"visibility": "hidden"
}
},
{
"name": "common_userdefine_app_name",
"label": "User Define APP Name",
"type": "string",
"doc": {
"visibility": "hidden"
}
},
{
"name": "common_app_surrogate_id",
"type": "string",
"label": "Surrogate ID",
"doc": {
"visibility": "hidden"
}
},
{
"name": "common_l7_protocol",
"type": "string",
"label": "L7 Protocol"
},
{
"name": "common_service_category",
"label": "FQDN Category",
"type": {
"type": "array",
"items": "int"
}
},
{
"name": "common_start_time",
"type": "long",
"doc": {
"constraints": {
"type": "timestamp"
},
"visibility": "hidden"
},
"label": "Start Time"
},
{
"name": "common_end_time",
"type": "long",
"doc": {
"constraints": {
"type": "timestamp"
},
"visibility": "hidden"
},
"label": "End Time"
},
{
"name": "common_establish_latency_ms",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Establish Latency(ms)"
},
{
"name": "common_con_duration_ms",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Duration(ms)"
},
{
"name": "common_stream_dir",
"type": "int",
"doc": {
"data": [
{
"code": "1",
"value": "c2s"
},
{
"code": "2",
"value": "s2c"
},
{
"code": "3",
"value": "double"
}
]
},
"label": "Stream Direction"
},
{
"name": "common_address_list",
"type": "string",
"doc": {
"visibility": "disabled"
},
"label": "Address List"
},
{
"name": "common_has_dup_traffic",
"type": "int",
"doc": {
"data": [
{
"code": "0",
"value": "No"
},
{
"code": "1",
"value": "Yes"
}
],
"visibility": "hidden"
},
"label": "Duplication Traffic"
},
{
"name": "common_stream_error",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Stream Error"
},
{
"name": "common_stream_trace_id",
"type": "long",
"doc": {
"allow_query": "true"
},
"label": "Session ID"
},
{
"name": "common_link_info_c2s",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Link Info(c2s)"
},
{
"name": "common_link_info_s2c",
"type": "string",
"doc": {
"visibility": "hidden"
},
"label": "Link Info(s2c)"
},
{
"name": "common_c2s_ipfrag_num",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Fragmentation Packets(c2s)"
},
{
"name": "common_s2c_ipfrag_num",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Fragmentation Packets(s2c)"
},
{
"name": "common_c2s_tcp_lostlen",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Sequence Gap Loss(c2s)"
},
{
"name": "common_s2c_tcp_lostlen",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Sequence Gap Loss(s2c)"
},
{
"name": "common_c2s_tcp_unorder_num",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Unorder Packets(c2s)"
},
{
"name": "common_s2c_tcp_unorder_num",
"type": "long",
"doc": {
"visibility": "hidden"
},
"label": "Unorder Packets(s2c)"
},
{
"name": "common_c2s_pkt_retrans",
"type": "long",
"label": "Packet Retransmission(c2s)"
},
{
"name": "common_s2c_pkt_retrans",
"type": "long",
"label": "Packet Retransmission(s2c)"
},
{
"name": "common_c2s_byte_retrans",
"type": "long",
"label": "Byte Retransmission(c2s)"
},
{
"name": "common_s2c_byte_retrans",
"type": "long",
"label": "Byte Retransmission(s2c)"
},
{
"name": "common_tcp_client_isn",
"label": "TCP Client ISN",
"doc": {
"visibility": "disabled"
},
"type": "long"
},
{
"name": "common_tcp_server_isn",
"label": "TCP Server ISN",
"doc": {
"visibility": "disabled"
},
"type": "long"
},
{
"name": "common_first_ttl",
"type": "int",
"doc": {
"visibility": "hidden"
},
"label": "First TTL"
},
{
"name": "common_processing_time",
"type": "long",
"doc": {
"constraints": {
"type": "timestamp"
}
},
"label": "Processing Time"
},
{
"name": "common_mirrored_pkts",
"label": "Mirrored Packets",
"type": "long",
"doc": {
"visibility": "hidden"
}
},
{
"name": "common_mirrored_bytes",
"label": "Mirrored Bytes",
"type": "long",
"doc": {
"visibility": "hidden"
}
},
{
"name": "nic_name",
"type": "string",
"label": "Nic Name"
},
{
"name": "origin_source_mac",
"type": "string",
"label": "Origin Source Mac"
},
{
"name": "origin_dest_mac",
"type": "string",
"label": "Origin Dest Mac"
},
{
"name": "packet_url",
"type": "string",
"label": "Packet URL"
},
{
"name": "pcap_storage_task_id",
"type": "int",
"label": "Task ID"
},
{
"name": "pcap_storage_duration",
"type": "int",
"label": "Duration"
}
]
}

38
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_storage_log.json Normal file
View File

@@ -0,0 +1,38 @@
{
"type": "record",
"name": "sys_storage_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "log_type",
"type": "string"
},
{
"name": "data_center",
"type": "string"
},
{
"name": "max_size",
"type": "long"
},
{
"name": "used_size",
"type": "long"
},
{
"name": "aggregate_size",
"type": "long"
},
{
"name": "last_storage",
"type": "long"
}
]
}

11
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/tables_cluster.json Normal file
View File

@@ -0,0 +1,11 @@
{
"namespace": "system",
"type": "record",
"name": "tables_cluster",
"fields": [
{
"name": "database",
"type": "string"
}
]
}

42
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_client_ip_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_client_ip_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "source",
"type": "string"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
}
]
}

42
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_external_host_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_external_host_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "destination",
"type": "string"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
}
]
}

42
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_internal_host_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_internal_host_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "source",
"type": "string"
}
]
}

42
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_server_ip_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_server_ip_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "destination",
"type": "string"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
}
]
}

22
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_urls_log.json Normal file
View File

@@ -0,0 +1,22 @@
{
"type": "record",
"name": "top_urls_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "url",
"type": "string"
}
]
}

42
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_user_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_user_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "subscriber_id",
"type": "string"
}
]
}

42
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_website_domain_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "top_website_domain_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "domain",
"type": "string"
},
{
"name": "order_by",
"type": "string"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "session_num",
"type": "long"
}
]
}

42
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_app_stat_log.json Normal file
View File

@@ -0,0 +1,42 @@
{
"type": "record",
"name": "traffic_app_stat_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "app_name",
"type": "string"
},
{
"name": "sub_app_name",
"type": "string"
},
{
"name": "session_num",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
}
]
}

218
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_metrics_log.json Normal file
View File

@@ -0,0 +1,218 @@
{
"type": "record",
"name": "traffic_metrics_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "allow_conn_num",
"type": "long"
},
{
"name": "allow_in_bytes",
"type": "long"
},
{
"name": "allow_in_packets",
"type": "long"
},
{
"name": "allow_out_bytes",
"type": "long"
},
{
"name": "allow_out_packets",
"type": "long"
},
{
"name": "close_conn_num",
"type": "long"
},
{
"name": "default_conn_num",
"type": "long"
},
{
"name": "default_in_bytes",
"type": "long"
},
{
"name": "default_in_packets",
"type": "long"
},
{
"name": "default_out_bytes",
"type": "long"
},
{
"name": "default_out_packets",
"type": "long"
},
{
"name": "deny_conn_num",
"type": "long"
},
{
"name": "deny_in_bytes",
"type": "long"
},
{
"name": "deny_in_packets",
"type": "long"
},
{
"name": "deny_out_bytes",
"type": "long"
},
{
"name": "deny_out_packets",
"type": "long"
},
{
"name": "device_id",
"type": "string"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "intercept_conn_num",
"type": "long"
},
{
"name": "intercept_in_bytes",
"type": "long"
},
{
"name": "intercept_in_packets",
"type": "long"
},
{
"name": "intercept_out_bytes",
"type": "long"
},
{
"name": "intercept_out_packets",
"type": "long"
},
{
"name": "established_conn_num",
"type": "long"
},
{
"name": "monitor_conn_num",
"type": "long"
},
{
"name": "monitor_in_bytes",
"type": "long"
},
{
"name": "monitor_in_packets",
"type": "long"
},
{
"name": "monitor_out_bytes",
"type": "long"
},
{
"name": "monitor_out_packets",
"type": "long"
},
{
"name": "new_conn_num",
"type": "long"
},
{
"name": "total_in_bytes",
"type": "long"
},
{
"name": "total_in_packets",
"type": "long"
},
{
"name": "total_out_bytes",
"type": "long"
},
{
"name": "total_out_packets",
"type": "long"
},
{
"name": "alert_bytes",
"type": "long"
},
{
"name": "hijk_bytes",
"type": "long"
},
{
"name": "ins_bytes",
"type": "long"
},
{
"name": "intcp_allow_num",
"type": "long"
},
{
"name": "intcp_deny_num",
"type": "long"
},
{
"name": "intcp_hijk_num",
"type": "long"
},
{
"name": "intcp_ins_num",
"type": "long"
},
{
"name": "intcp_mon_num",
"type": "long"
},
{
"name": "intcp_rdirt_num",
"type": "long"
},
{
"name": "intcp_repl_num",
"type": "long"
},
{
"name": "maybe_pinning_num",
"type": "long"
},
{
"name": "not_pinning_num",
"type": "long"
},
{
"name": "pinning_num",
"type": "long"
},
{
"name": "ad_cc_bytes",
"type": "long"
},
{
"name": "ad_flood_bytes",
"type": "long"
},
{
"name": "ad_reflection_bytes",
"type": "long"
},
{
"name": "intcp_edit_elem_num",
"type": "long"
}
]
}

78
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_protocol_stat_log.json Normal file
View File

@@ -0,0 +1,78 @@
{
"type": "record",
"name": "traffic_protocol_stat_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "protocol_id",
"type": "string"
},
{
"name": "isp",
"type": "string"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "data_center",
"type": "string"
},
{
"name": "device_group",
"type": "string"
},
{
"name": "sessions",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "c2s_ipfrag_num",
"type": "long"
},
{
"name": "s2c_ipfrag_num",
"type": "long"
},
{
"name": "c2s_tcp_lostlen",
"type": "long"
},
{
"name": "s2c_tcp_lostlen",
"type": "long"
},
{
"name": "c2s_tcp_unorder_num",
"type": "long"
},
{
"name": "s2c_tcp_unorder_num",
"type": "long"
}
]
}

78
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_summary_log.json Normal file
View File

@@ -0,0 +1,78 @@
{
"type": "record",
"name": "traffic_summary_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "isp",
"type": "string"
},
{
"name": "entrance_id",
"type": "long"
},
{
"name": "device_group",
"type": "string"
},
{
"name": "data_center",
"type": "string"
},
{
"name": "schema_type",
"type": "string"
},
{
"name": "ip_object",
"type": "string"
},
{
"name": "sessions",
"type": "long"
},
{
"name": "c2s_pkt_num",
"type": "long"
},
{
"name": "s2c_pkt_num",
"type": "long"
},
{
"name": "c2s_byte_num",
"type": "long"
},
{
"name": "s2c_byte_num",
"type": "long"
},
{
"name": "one_sided_connections",
"type": "long"
},
{
"name": "uncategorized_bytes",
"type": "long"
},
{
"name": "fragmentation_packets",
"type": "long"
},
{
"name": "sequence_gap_loss",
"type": "long"
},
{
"name": "unorder_packets",
"type": "long"
}
]
}

46
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_top_destination_ip_metrics_log.json Normal file
View File

@@ -0,0 +1,46 @@
{
"type": "record",
"name": "traffic_top_destination_ip_metrics_log",
"namespace": "druid",
"doc": {
"partition_key": "__time"
},
"fields": [
{
"name": "__time",
"type": "long"
},
{
"name": "common_sled_ip",
"type": "string"
},
{
"name": "common_data_center",
"type": "string"
},
{
"name": "destination_ip",
"type": "string"
},
{
"name": "attack_type",
"type": "string"
},
{
"name": "session_rate",
"type": "long"
},
{
"name": "packet_rate",
"type": "long"
},
{
"name": "bit_rate",
"type": "long"
},
{
"name": "partition_num",
"type": "long"
}
]
}

5
TSG发布版本更新记录/TSG-21.12/qgw/schema/transaction_record.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/transaction_record.json
View File

@@ -255,7 +255,8 @@
"internal_columns": [
"common_recv_time",
"common_log_id",
"common_processing_time"
"common_processing_time",
"common_tunnels"
],
"tunnel_type": {
"$ref": "public_schema_info.json#/tunnel_type"
@@ -719,7 +720,7 @@
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
},
"allow_query": "true"
"visibility": "hidden"
},
"type": "string"
},

95
TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/version.json Normal file
View File

@@ -0,0 +1,95 @@
{
"product": "185 Environment",
"version": "21.11",
"registered": "Geedge",
"updated": "2021-08-01 00:00:00",
"components": {
"oss": [
{
"name": "zookeeper",
"version": "3.4.10",
"licenseType": "Apache License 2.0",
"description": "分布式应用程序协调服务"
},
{
"name": "kafka",
"version": "2.11_1.0.0",
"licenseType": "Apache License 2.0",
"description": "消息队列"
},
{
"name": "habse",
"version": "2.2.3",
"licenseType": "Apache License 2.0",
"description": "用于文件系统和存储Radius数据"
},
{
"name": "flume",
"version": "1.9.0",
"licenseType": "Apache License 2.0",
"description": "日志补全传输"
},
{
"name": "clickhouse",
"version": "20.3.12.112-cluster",
"licenseType": "Apache License 2.0",
"description": "原始日志数据库"
},
{
"name": "druid",
"version": "0.18.1",
"licenseType": "Apache License 2.0",
"description": "分析实时数据并提供低延迟查询的OLAP应用程序"
},
{
"name": "gohangout",
"version": "1.15.2.20210408",
"description": "动态获取原始日志表schema入库程序"
}
],
"apps": [
{
"name": "galaxy-qgw-service",
"version": "345",
"description": "数据平台对外统一查询网关"
},
{
"name": "galaxy-report-service",
"version": "21.04.07",
"description": "自定义报表查询服务"
},
{
"name": "galaxy-hos-service",
"version": "21.07.01",
"description": "对象存储服务"
},
{
"name": "xxl-job-admin",
"version": "v1.3.20210408",
"description": "分布式任务调度平台"
},
{
"name": "xxl-job",
"version": "v1.3.210413-rc1",
"description": "分布式任务调度平台-执行器"
}
],
"tasks": [
{
"name": "flume",
"version": "flume-config-20.08",
"description": "原始日志补全、subscriber更新、Radius上下线功能"
},
{
"name": "druid",
"version": "druid-config-20.08",
"description": "所有分析日志任务"
},
{
"name": "gohangout",
"version": "gohangout-config-20.08",
"description": "原始日志入库、上下线日志入库"
}
]
}
}

2
TSG发布版本更新记录/TSG-21.12/qgw/schema/voip_record.json → TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/voip_record.json
View File

@@ -703,7 +703,7 @@
"key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
"value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
},
"allow_query": "true"
"visibility": "hidden"
},
"type": "string"
},

48
TSG发布版本更新记录/TSG-21.12/qgw/push_config.sh Normal file
View File

@@ -0,0 +1,48 @@
#!/bin/bash
#修改config_path_root路径为放置要上传nacos的配置，IP地址为nacos地址
config_path_root=/home/deploy/bak/galaxy-qgw-service/config
tenant="prod"
nacos_push_url="http://192.168.44.67:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy"
retry=3
#1.获取config 路径
config_path_list=$(readlink -f $config_path_root)
function pushConfig(){
#2.app文件夹文件
for config_path in $config_path_list
do
if [ -d $config_path ];then
appPath=${config_path%/*}
appName=${appPath##*/}
for file in `ls $config_path`
do
config_path_file=$config_path/$file
if [ -f $config_path_file ];then
data_id=${config_path_file##*/}
suffix=${config_path_file##*.}
if [ "$suffix" = "sql" ];then
suffix="text"
fi
if [ "$suffix" = "yml" ];then
suffix="yaml"
fi
for (( i = 0; i < 3; i++ )); do
if [ "$suffix" = "yaml" ]||[ "$suffix" = "json" ]||[ "$suffix" = "text" ]||[ "$suffix" = "properties" ];then
result=$(curl -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$appName&type=$suffix" --data-urlencode content="`cat $config_path_file`")
echo "push config response code "$result
if [[ $result -eq '200' ]];then
i=3
fi
fi
done
fi
done
fi
done
}
pushConfig

20
TSG发布版本更新记录/TSG-21.12/qgw/update-galaxy-qgw-service.yml Normal file
View File

@@ -0,0 +1,20 @@
#在galaxy-qgw-service.yml 配置最下增加以下配置
## job cfg
job:
interactive:
timeout: 30000
response:
timeout: 500
execution:
timeout: 300000
pool:
corePoolSize: 20
maxPoolSize: 30
queueCapacity: 10
## task cfg
task:
pool:
corePoolSize: 8
maxPoolSize: 16
queueCapacity: 1000

24
TSG发布版本更新记录/TSG-21.12/record-21.12.txt
View File

@@ -1,21 +1,24 @@
21.12更新记录，按顺序更新
21.12更新记录，按顺序更新，具体更新文件详见相关目录
kafka
1.新增topic TRAFFIC-APP-STAT
clickhouse
1.新增字段，clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password 进入ck客户端执行SQL文件里的SQL
2.修改clickhouse-server/config.xml中配置
hbase
1.新增hbase表
2.新增Phoenix视图
clickhouse
1.新增字段，进入ck客户端执行SQL文件里的SQL
druid
1.更新Apache Druid任务traffic_app_stat_log, traffic_metrics_log
2.druid所有_common/目录下增加log4j2.component.properties
网关
1.更新镜像350
2.更新schema 15个
2.更新schema,使用backup_config.sh备份，push_config.sh提交配置到nacos
预处理
1.更新补全jar包及配置
@@ -25,12 +28,19 @@ druid
5.新增vpn-recommend任务
hos
1.更新镜像21.12.01
1.更新镜像21.12.20
report
1.更新镜像21-12-15
flink
1.修改临时目录
2.conf目录增加log4j2.component.properties
spark
1.修改环境变量
1.修改临时目录
gohangout
1.更新config下所有任务配置max.partition.fetch.bytes: '31457280'
执行完成后重启所有gohangout

7
TSG发布版本更新记录/TSG-21.12/spark/update-spark-conf.txt
View File

@@ -1,2 +1,7 @@
#优化spark 使用脚本重启集群失败问题（pid存到/tmp目录）
spark-2.2.3-bin-hadoop2.7/conf/spark-env.sh
新增：export SPARK_PID_DIR=${SPARK_HOME}/tmp
新增以下配置项：
export SPARK_PID_DIR=${SPARK_HOME}/tmp

9
TSG发布版本更新记录/TSG-21.12/topology/README.txt Normal file
View File

@@ -0,0 +1,9 @@
flink任务更新步骤，顺序不分先后
注：修改各个更新的文本文件中ip地址等变量配置为线上实际值
启停任务使用各个目录下stop.sh ,start.sh
1.新增account-framedip-Hbase任务，停调accountHbase，frameip任务
2.更新dos-detection任务jar包，config/下配置新增update-config.txt中配置项
3.更新flink-top任务jar包及kafka-flinksql-top.sql文件
4.新增vpn-recommend任务

0
TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 → TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2
View File

0
TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/start.sh → TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/start.sh
View File

0
TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/stop.sh → TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/stop.sh
View File

4
TSG发布版本更新记录/TSG-21.12/topology/dos-detection/update-config Normal file
View File

@@ -0,0 +1,4 @@
# dos-detection任务config中配置新增以下配置
#baseline ttl，单位：天
hbase.baseline.ttl=30

50
TSG发布版本更新记录/TSG-21.12/flink/topN/kafka-flinksql-top.sql → TSG发布版本更新记录/TSG-21.12/topology/flink-top/kafka-flinksql-top.sql
View File

@@ -10,8 +10,8 @@ common_l4_protocol VARCHAR,
common_internal_ip VARCHAR,
common_external_ip VARCHAR,
common_subscriber_id VARCHAR,
common_app_label VARCHAR,
common_sessions BIGINT,
common_app_label VARCHAR,
common_c2s_pkt_num BIGINT,
common_s2c_pkt_num BIGINT,
common_c2s_byte_num BIGINT,
@@ -23,10 +23,10 @@ WITH(
'connector' = 'kafka',
'properties.group.id' = 'kafka-indexing-service',
'topic' = 'SESSION-RECORD-COMPLETED',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094',
'properties.security.protocol'='SASL_PLAINTEXT',
'properties.sasl.mechanism'='PLAIN',
'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'scan.startup.mode' = 'latest-offset',
'sink.parallelism'='1',
--'sink.parallelism'='60',
@@ -47,10 +47,10 @@ PRIMARY KEY (stat_time) NOT ENFORCED
)WITH(
'connector' = 'upsert-kafka',
'topic' = 'TOP-CLIENT-IP',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094',
'properties.security.protocol'='SASL_PLAINTEXT',
'properties.sasl.mechanism'='PLAIN',
'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
--'sink.parallelism'='1',
'key.format' = 'json',
'value.format' = 'json'
@@ -87,9 +87,6 @@ FROM
top_client_ip_view)
WHERE rownum <= 1000);
--server:
CREATE TABLE top_server_ip_log(
destination VARCHAR,
@@ -104,10 +101,10 @@ PRIMARY KEY (stat_time) NOT ENFORCED
)WITH(
'connector' = 'upsert-kafka',
'topic' = 'TOP-SERVER-IP',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094',
'properties.security.protocol'='SASL_PLAINTEXT',
'properties.sasl.mechanism'='PLAIN',
'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
--'sink.parallelism'='1',
'key.format' = 'json',
'value.format' = 'json'
@@ -144,7 +141,6 @@ FROM
top_server_ip_view)
WHERE rownum <= 1000);
--internal
CREATE TABLE top_internal_ip_log (
source VARCHAR,
@@ -159,10 +155,10 @@ CREATE TABLE top_internal_ip_log (
) WITH (
'connector' = 'upsert-kafka',
'topic' = 'TOP-INTERNAL-HOST',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094',
'properties.security.protocol'='SASL_PLAINTEXT',
'properties.sasl.mechanism'='PLAIN',
'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
--'sink.parallelism'='1',
'key.format' = 'json',
'value.format' = 'json'
@@ -215,10 +211,10 @@ CREATE TABLE top_external_ip_log (
) WITH (
'connector' = 'upsert-kafka',
'topic' = 'TOP-EXTERNAL-HOST',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094',
'properties.security.protocol'='SASL_PLAINTEXT',
'properties.sasl.mechanism'='PLAIN',
'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
--'sink.parallelism'='1',
'key.format' = 'json',
'value.format' = 'json'
@@ -257,7 +253,6 @@ FROM
top_common_external_ip_view)
WHERE rownum <= 1000);
--website_domain
CREATE TABLE top_website_domain_log (
domain VARCHAR,
@@ -272,10 +267,10 @@ CREATE TABLE top_website_domain_log (
) WITH (
'connector' = 'upsert-kafka',
'topic' = 'TOP-WEBSITE-DOMAIN',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094',
'properties.security.protocol'='SASL_PLAINTEXT',
'properties.sasl.mechanism'='PLAIN',
'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
--'sink.parallelism'='1',
'key.format' = 'json',
'value.format' = 'json'
@@ -328,10 +323,10 @@ CREATE TABLE top_user_log (
) WITH (
'connector' = 'upsert-kafka',
'topic' = 'TOP-USER',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094',
'properties.security.protocol'='SASL_PLAINTEXT',
'properties.sasl.mechanism'='PLAIN',
'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
--'sink.parallelism'='1',
'key.format' = 'json',
'value.format' = 'json'
@@ -368,10 +363,6 @@ FROM
top_user_log_view)
WHERE rownum <= 1000);
--app
CREATE TABLE top_app_log (
app_name VARCHAR,
@@ -385,11 +376,10 @@ CREATE TABLE top_app_log (
) WITH (
'connector' = 'upsert-kafka',
'topic' = 'TRAFFIC-APP-STAT',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094',
--'properties.bootstrap.servers' = '10.111.136.193:9092,10.111.136.194:9092,10.111.136.195:9092,10.111.136.196:9092,10.111.136.197:9092,10.111.136.198:9092,10.111.136.199:9092,10.111.136.200:9092,10.111.136.201:9092,10.111.136.203:9092,10.111.136.204:9092,10.111.136.205:9092,10.111.136.206:9092,10.111.136.207:9092,10.111.136.202:9092',
'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094,192.168.44.16:9094',
'properties.security.protocol'='SASL_PLAINTEXT',
'properties.sasl.mechanism'='PLAIN',
'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";',
'key.format' = 'json',
'value.format' = 'json'
);
@@ -407,9 +397,3 @@ INSERT INTO top_app_log
ROW_NUMBER() OVER (PARTITION BY stat_time )
FROM
top_app_log_view));

2
TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/config/RECOMMENDATION-APP-CIP → TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/config/RECOMMENDATION-APP-CIP
View File

@@ -31,7 +31,7 @@ kafka.user=admin
#kafka SASL及SSL验证密码
kafka.pin=galaxy2019
#1SSL需要
tools.library=D:\\K18-Phase2\\tsgSpace\\dat\\tsg\\
tools.library=/home/bigdata/topology/dat/
#是否接受全量app 无过滤条件false 白名单过滤true
has.filter=false
#只计算filter命中的common_app_label逗号分隔 baidu.com,qq 可不填写

3
TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/start.sh → TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/start.sh
View File

@@ -1,5 +1,4 @@
#! /bin/bash
#启动storm任务脚本
#!/bin/bash
source /etc/profile
#任务jar所在目录
BASE_DIR=`pwd`

0
TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/stop.sh → TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/stop.sh
View File