From f4cee9dd8c7913fbdfc1bd0c008fdf389c972f2f Mon Sep 17 00:00:00 2001 From: zhanghongqing Date: Tue, 21 Dec 2021 17:04:24 +0800 Subject: [PATCH] 21.12 update --- .../druid/{update-druid-2112.sql => README.txt} | 50 +- .../TSG-21.12/druid/log4j2.component.properties | 1 + .../service_flow_config.properties | 24 - TSG发布版本更新记录/TSG-21.12/flink/config.sh | 3 + .../TSG-21.12/flink/dos-detection/common.properties | 5 - .../TSG-21.12/flink/log4j2.component.properties | 1 + .../TSG-21.12/flink/vpn-recommend/common.properties | 38 - .../TSG-21.12/hbase/update-hbase.txt | 11 +- .../TSG-21.12/hbase/update-phoenix.txt | 250 +++++- .../TSG-21.12/kafka/add-topic.txt | 2 +- TSG发布版本更新记录/TSG-21.12/qgw/README.txt | 6 + .../TSG-21.12/qgw/backup_config.sh | 31 + .../config/active_defence_event.json | 322 +++++++ .../qgw/galaxy-qgw-service/config/ck-filter.json | 99 +++ .../galaxy-qgw-service/config/ck-queries-template.sql | 118 +++ .../galaxy-qgw-service/config/columns_cluster.json | 11 + .../qgw/galaxy-qgw-service/config/disks_cluster.json | 11 + .../config}/dos_event.json | 6 +- .../qgw/galaxy-qgw-service/config/druid-filter.json | 21 + .../config/druid-queries-template.sql | 92 ++ .../qgw/galaxy-qgw-service/config/engine-filter.json | 53 ++ .../config}/engine-queries-template.sql | 0 .../config}/gtpc_record.json | 2 +- .../config}/hbase-filter.json | 0 .../config}/hbase-queries-template.sql | 0 .../config}/interim_session_record.json | 6 +- .../qgw/galaxy-qgw-service/config/job_result.json | 7 + .../galaxy-qgw-service/config/liveChart_interim.json | 169 ++++ .../galaxy-qgw-service/config/liveChart_session.json | 169 ++++ .../config}/meta_data.json | 4 +- .../qgw/galaxy-qgw-service/config/parts_cluster.json | 11 + .../qgw/galaxy-qgw-service/config/processes.json | 11 + .../config}/proxy_event.json | 6 +- .../config/proxy_event_hits_log.json | 58 ++ .../config}/public_schema_info.json | 2 +- .../qgw/galaxy-qgw-service/config/query_log.json | 11 + .../galaxy-qgw-service/config/query_log_cluster.json | 11 + .../galaxy-qgw-service/config/radius_onff_log.json | 37 + .../config}/radius_record.json | 2 +- .../config}/recommendation_app_cip.json | 0 .../config/relation_account_framedip.json | 7 + .../qgw/galaxy-qgw-service/config/report_result.json | 7 + .../config}/security_event.json | 6 +- .../config/security_event_hits_log.json | 42 + .../config}/session_record.json | 6 +- .../config/session_record_common_client_ip.json | 71 ++ .../config/session_record_common_server_ip.json | 71 ++ .../config/session_record_http_domain.json | 71 ++ .../config/sys_packet_capture_event.json | 801 ++++++++++++++++++ .../galaxy-qgw-service/config/sys_storage_log.json | 38 + .../qgw/galaxy-qgw-service/config/tables_cluster.json | 11 + .../galaxy-qgw-service/config/top_client_ip_log.json | 42 + .../config/top_external_host_log.json | 42 + .../config/top_internal_host_log.json | 42 + .../galaxy-qgw-service/config/top_server_ip_log.json | 42 + .../qgw/galaxy-qgw-service/config/top_urls_log.json | 22 + .../qgw/galaxy-qgw-service/config/top_user_log.json | 42 + .../config/top_website_domain_log.json | 42 + .../config/traffic_app_stat_log.json | 42 + .../config/traffic_metrics_log.json | 218 +++++ .../config/traffic_protocol_stat_log.json | 78 ++ .../config/traffic_summary_log.json | 78 ++ .../traffic_top_destination_ip_metrics_log.json | 46 + .../config}/transaction_record.json | 5 +- .../qgw/galaxy-qgw-service/config/version.json | 95 +++ .../config}/voip_record.json | 2 +- .../TSG-21.12/qgw/push_config.sh | 48 ++ .../TSG-21.12/qgw/update-galaxy-qgw-service.yml | 20 + .../TSG-21.12/record-21.12.txt | 24 +- .../TSG-21.12/spark/update-spark-conf.txt | 7 +- .../TSG-21.12/topology/README.txt | 9 + .../config/RADIUS-RELATIONSHIP-HBASE-V2 | 0 .../account-framedip-Hbase/start.sh | 0 .../account-framedip-Hbase/stop.sh | 0 .../TSG-21.12/topology/dos-detection/update-config | 4 + .../flink-top}/kafka-flinksql-top.sql | 50 +- .../vpn-recommend/config/RECOMMENDATION-APP-CIP | 2 +- .../{flink => topology}/vpn-recommend/start.sh | 3 +- .../{flink => topology}/vpn-recommend/stop.sh | 0 79 files changed, 3655 insertions(+), 142 deletions(-) rename TSG发布版本更新记录/TSG-21.12/druid/{update-druid-2112.sql => README.txt} (75%) create mode 100644 TSG发布版本更新记录/TSG-21.12/druid/log4j2.component.properties delete mode 100644 TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/config.sh delete mode 100644 TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties create mode 100644 TSG发布版本更新记录/TSG-21.12/flink/log4j2.component.properties delete mode 100644 TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/README.txt create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/backup_config.sh create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/active_defence_event.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-filter.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-queries-template.sql create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/columns_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/disks_cluster.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/dos_event.json (98%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-filter.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-queries-template.sql create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/engine-filter.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/engine-queries-template.sql (100%) rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/gtpc_record.json (99%) rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/hbase-filter.json (100%) rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/hbase-queries-template.sql (100%) rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/interim_session_record.json (99%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/job_result.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_interim.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_session.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/meta_data.json (97%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/parts_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/processes.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/proxy_event.json (99%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event_hits_log.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/public_schema_info.json (99%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_onff_log.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/radius_record.json (99%) rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/recommendation_app_cip.json (100%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/relation_account_framedip.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/report_result.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/security_event.json (99%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event_hits_log.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/session_record.json (99%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_client_ip.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_server_ip.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_http_domain.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_packet_capture_event.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_storage_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/tables_cluster.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_client_ip_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_external_host_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_internal_host_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_server_ip_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_urls_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_user_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_website_domain_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_app_stat_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_metrics_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_protocol_stat_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_summary_log.json create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_top_destination_ip_metrics_log.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/transaction_record.json (99%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/version.json rename TSG发布版本更新记录/TSG-21.12/qgw/{schema => galaxy-qgw-service/config}/voip_record.json (99%) create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/push_config.sh create mode 100644 TSG发布版本更新记录/TSG-21.12/qgw/update-galaxy-qgw-service.yml create mode 100644 TSG发布版本更新记录/TSG-21.12/topology/README.txt rename TSG发布版本更新记录/TSG-21.12/{flink => topology}/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 (100%) rename TSG发布版本更新记录/TSG-21.12/{flink => topology}/account-framedip-Hbase/start.sh (100%) rename TSG发布版本更新记录/TSG-21.12/{flink => topology}/account-framedip-Hbase/stop.sh (100%) create mode 100644 TSG发布版本更新记录/TSG-21.12/topology/dos-detection/update-config rename TSG发布版本更新记录/TSG-21.12/{flink/topN => topology/flink-top}/kafka-flinksql-top.sql (88%) rename TSG发布版本更新记录/TSG-21.12/{flink => topology}/vpn-recommend/config/RECOMMENDATION-APP-CIP (95%) rename TSG发布版本更新记录/TSG-21.12/{flink => topology}/vpn-recommend/start.sh (95%) rename TSG发布版本更新记录/TSG-21.12/{flink => topology}/vpn-recommend/stop.sh (100%) diff --git a/TSG发布版本更新记录/TSG-21.12/druid/update-druid-2112.sql b/TSG发布版本更新记录/TSG-21.12/druid/README.txt similarity index 75% rename from TSG发布版本更新记录/TSG-21.12/druid/update-druid-2112.sql rename to TSG发布版本更新记录/TSG-21.12/druid/README.txt index 53915e0..e48fd0b 100644 --- a/TSG发布版本更新记录/TSG-21.12/druid/update-druid-2112.sql +++ b/TSG发布版本更新记录/TSG-21.12/druid/README.txt @@ -1,20 +1,54 @@ -#更新一 traffic_app_stat_log -# 更新的时候，先停掉之前的任务，然后执行sql，最后再提交新任务 +druid 任务更新步骤 +概述 +1.先停掉之前的任务，然后执行sql，最后再提交新任务 +2.根据实际情况执行集群或单机版操作 + +操作 +一.更新traffic_app_stat_log +1.登录druid提交任务服务器（通常为第一台）进入 */druid_topology/rule/ +2.停止需要更新的任务 ./supervisor-manger terminate traffic_app_stat_log ， ./supervisor-manger terminate traffic_metrics_log +3. 修改traffic_app_stat_log.json traffic_metrics_log.json中IP地址后，复制到 */druid_topology/tasks下 + +4.登录galaxy使用的mariadb，使用druid数据库执行以下SQL +执行SQL一： DELETE FROM druid.druid_pendingsegments WHERE dataSource = 'traffic_app_stat_log'; - DELETE FROM druid.druid_datasource WHERE dataSource = 'traffic_app_stat_log'; - - -#更新二 traffic_metrics_log(不要在widows环境解压操作) +执行SQL二： 单机版sql: INSERT INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES('traffic_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-12-12T12:32:13.207Z', 'traffic_metrics_log', '2021-12-12T12:32:14.239Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-12-12T12:32:13.207Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6D6574726963735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D31322D31325431323A33323A31332E3230375A222C226C6F616453706563223A7B2274797065223A226C6F63616C222C2270617468223A222F6170616368652D64727569642D302E31382E312F7661722F64727569642F7365676D656E74732F747261666669635F6D6574726963735F6C6F672F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032312D31322D31325431323A33323A31332E3230375A2F302F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226465766963655F69642C616C6C6F775F636F6E6E5F6E756D2C616C6C6F775F696E5F62797465732C616C6C6F775F696E5F7061636B6574732C616C6C6F775F6F75745F62797465732C616C6C6F775F6F75745F7061636B6574732C636C6F73655F636F6E6E5F6E756D2C64656661756C745F636F6E6E5F6E756D2C64656661756C745F696E5F62797465732C64656661756C745F696E5F7061636B6574732C64656661756C745F6F75745F62797465732C64656661756C745F6F75745F7061636B6574732C64656E795F636F6E6E5F6E756D2C64656E795F696E5F62797465732C64656E795F696E5F7061636B6574732C64656E795F6F75745F62797465732C64656E795F6F75745F7061636B6574732C65737461626C69736865645F636F6E6E5F6E756D2C696E746572636570745F636F6E6E5F6E756D2C696E746572636570745F696E5F62797465732C696E746572636570745F696E5F7061636B6574732C696E746572636570745F6F75745F62797465732C696E746572636570745F6F75745F7061636B6574732C6D6F6E69746F725F636F6E6E5F6E756D2C6D6F6E69746F725F696E5F62797465732C6D6F6E69746F725F696E5F7061636B6574732C6D6F6E69746F725F6F75745F62797465732C6D6F6E69746F725F6F75745F7061636B6574732C6E65775F636F6E6E5F6E756D2C746F74616C5F696E5F62797465732C746F74616C5F696E5F7061636B6574732C746F74616C5F6F75745F62797465732C656E7472616E63655F69642C746F74616C5F6F75745F7061636B6574732C70696E6E696E675F6E756D2C6E6F745F70696E6E696E675F6E756D2C6D617962655F70696E6E696E675F6E756D2C616C6572745F62797465732C696E735F62797465732C68696A6B5F62797465732C61645F7265666C656374696F6E5F62797465732C61645F666C6F6F645F62797465732C61645F63635F62797465732C696E7463705F6D6F6E5F6E756D2C696E7463705F616C6C6F775F6E756D2C696E7463705F64656E795F6E756D2C696E7463705F72646972745F6E756D2C696E7463705F7265706C5F6E756D2C696E7463705F68696A6B5F6E756D2C696E7463705F696E735F6E756D2C696E7463705F656469745F656C656D5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A31313633342C226964656E746966696572223A22747261666669635F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D31322D31325431323A33323A31332E3230375A227D); -单机版文件：standalone/traffic_metrics_log.zip 集群版sql: INSERT INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES('traffic_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-12-13T02:46:13.726Z', 'traffic_metrics_log', '2021-12-13T02:46:14.984Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-12-13T02:46:13.726Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6D6574726963735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D31322D31335430323A34363A31332E3732365A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F6D6574726963735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D31322D31335430325F34365F31332E3732365A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226465766963655F69642C616C6C6F775F636F6E6E5F6E756D2C616C6C6F775F696E5F62797465732C616C6C6F775F696E5F7061636B6574732C616C6C6F775F6F75745F62797465732C616C6C6F775F6F75745F7061636B6574732C636C6F73655F636F6E6E5F6E756D2C64656661756C745F636F6E6E5F6E756D2C64656661756C745F696E5F62797465732C64656661756C745F696E5F7061636B6574732C64656661756C745F6F75745F62797465732C64656661756C745F6F75745F7061636B6574732C64656E795F636F6E6E5F6E756D2C64656E795F696E5F62797465732C64656E795F696E5F7061636B6574732C64656E795F6F75745F62797465732C64656E795F6F75745F7061636B6574732C65737461626C69736865645F636F6E6E5F6E756D2C696E746572636570745F636F6E6E5F6E756D2C696E746572636570745F696E5F62797465732C696E746572636570745F696E5F7061636B6574732C696E746572636570745F6F75745F62797465732C696E746572636570745F6F75745F7061636B6574732C6D6F6E69746F725F636F6E6E5F6E756D2C6D6F6E69746F725F696E5F62797465732C6D6F6E69746F725F696E5F7061636B6574732C6D6F6E69746F725F6F75745F62797465732C6D6F6E69746F725F6F75745F7061636B6574732C6E65775F636F6E6E5F6E756D2C746F74616C5F696E5F62797465732C746F74616C5F696E5F7061636B6574732C746F74616C5F6F75745F62797465732C656E7472616E63655F69642C746F74616C5F6F75745F7061636B6574732C70696E6E696E675F6E756D2C6E6F745F70696E6E696E675F6E756D2C6D617962655F70696E6E696E675F6E756D2C616C6572745F62797465732C696E735F62797465732C68696A6B5F62797465732C61645F7265666C656374696F6E5F62797465732C61645F666C6F6F645F62797465732C61645F63635F62797465732C696E7463705F6D6F6E5F6E756D2C696E7463705F616C6C6F775F6E756D2C696E7463705F64656E795F6E756D2C696E7463705F72646972745F6E756D2C696E7463705F7265706C5F6E756D2C696E7463705F68696A6B5F6E756D2C696E7463705F696E735F6E756D2C696E7463705F656469745F656C656D5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A31313633342C226964656E746966696572223A22747261666669635F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D31322D31335430323A34363A31332E3732365A227D); -集群版文件路径： cluster/traffic_metrics_log.zip +5.更新索引文件 +#解压后替换druid目录下的文件(不要在widows环境解压操作) +单机版文件：standalone/traffic_metrics_log.zip 解压后替换druid目录下的文件var/druid/segments/ +#集群版需要上传到hdfs +集群版文件： cluster/traffic_metrics_log.zip 解压到hdfs服务器上后运行以下命令 +hdfs dfs -put traffic_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2021-12-13T02_46_13.726Z/ /druid/segments/traffic_metrics_log/30000101T000000.000Z_30000102T000000.000Z +6.启动任务，druid服务器上 +druid_topology/目录下 +使用以下命令提交 +变量：${var} traffic_app_stat_log.json，traffic_metrics_log.json +变量：${druid_host} druid服务器地址 + +curl -X 'POST' -H 'Content-Type:application/json' -d @${var} http://$druid_host:8081/druid/indexer/v1/supervisor + +7.修复log4j2漏洞 +集群版： +将log4j2.component.properties文件放在所有druid安装目录conf/druid/cluster/_common/ +单机版： +将log4j2.component.properties文件放在conf/druid/single-server/small|medium|large/_common/ +*需要重启所有druid服务 + +8.验证 +执行状态为RUNNING则任务正常 +curl http://${druid_host}:8089/druid/indexer/v1/supervisor?state=true |jq + +9.任务异常处理 +1.登录druid提交任务服务器（通常为第一台）进入 */druid_topology/rule/ +2. 执行./supervisor-manger reset 任务名称 ，例如./supervisor-manger reset traffic_app_stat_log \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/druid/log4j2.component.properties b/TSG发布版本更新记录/TSG-21.12/druid/log4j2.component.properties new file mode 100644 index 0000000..164621d --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/druid/log4j2.component.properties @@ -0,0 +1 @@ +log4j2.formatMsgNoLookups=true diff --git a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties b/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties deleted file mode 100644 index 17dc226..0000000 --- a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/service_flow_config.properties +++ /dev/null @@ -1,24 +0,0 @@ -#管理kafka地址 -input.kafka.servers=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094 -#input.kafka.servers=192.168.44.12:9094 - -#hbase zookeeper地址 用于连接HBase -hbase.zookeeper.servers=192.168.44.11,192.168.44.14,192.168.44.15 -#hbase.zookeeper.servers=192.168.44.11:2181 - -#--------------------------------Kafka消费组信息------------------------------# - -#kafka 接收数据topic -input.kafka.topic=RADIUS-RECORD - -#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； -group.id=radius-flink-20211124 - -#--------------------------------topology配置------------------------------# -#ip-account对应关系表 -hbase.framedip.table.name=tsg_galaxy:relation_framedip_account - -#定位库地址 -tools.library=/home/bigdata/topology/dat/ -#account-ip对应关系表 -hbase.account.table.name=tsg_galaxy:relation_account_framedip diff --git a/TSG发布版本更新记录/TSG-21.12/flink/config.sh b/TSG发布版本更新记录/TSG-21.12/flink/config.sh new file mode 100644 index 0000000..311500d --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/config.sh @@ -0,0 +1,3 @@ +#flink bin/config.sh中临时目录设置，用于存放进程id + +DEFAULT_ENV_PID_DIR="$(cd "`dirname "$0"`"/..; pwd)/tmp" \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties b/TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties deleted file mode 100644 index 57b5c9c..0000000 --- a/TSG发布版本更新记录/TSG-21.12/flink/dos-detection/common.properties +++ /dev/null @@ -1,5 +0,0 @@ -# dos任务新增以下配置 - - -#baseline ttl，单位：天 -hbase.baseline.ttl=30 diff --git a/TSG发布版本更新记录/TSG-21.12/flink/log4j2.component.properties b/TSG发布版本更新记录/TSG-21.12/flink/log4j2.component.properties new file mode 100644 index 0000000..164621d --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/flink/log4j2.component.properties @@ -0,0 +1 @@ +log4j2.formatMsgNoLookups=true diff --git a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties b/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties deleted file mode 100644 index 9e96c4e..0000000 --- a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/common.properties +++ /dev/null @@ -1,38 +0,0 @@ -#kafka的地址信息 -source.kafka.broker=192.168.44.11:9094 -source.kafka.group.id =vpn-1206-1 -source.kafka.topic=SESSION-RECORD-COMPLETED -source.kafka.parallelism=1 -max.poll.records=3000 -session.timeout.ms=60000 -max.partition.fetch.bytes=31457280 -#hbase的zk地址 -zk.host=192.168.44.11:2181 -#写入hbase并行度 -sink.hbase.parallelism=1 -#写入hbase列簇 -sink.hbase.fm=common -#写入hbase表名 -sink.hbase.table=tsg_galaxy:recommendation_app_cip -#任务并行度 -task.parallelism=1 -#窗口延迟等待时间单位秒 -watermark.time=1 -#top结果限制 -top.limit=10000 -#滑动窗口总时间单位分钟 -slidingwindow.time.minute=30 -#每个滑块时间单位分钟 -slidingwindowslot.time.minute=1 -#kafka是否开启安全验证 0不开启 1SSL 2 SASL -kafka.security=2 -#kafka SASL验证用户名 -kafka.user=admin -#kafka SASL及SSL验证密码 -kafka.pin=galaxy2019 -#1SSL需要 -tools.library=D:\\K18-Phase2\\tsgSpace\\dat\\tsg\\ -#是否接受全量app 无过滤条件false 白名单过滤true -has.filter=false -#只计算filter命中的common_app_label逗号分隔 baidu.com,qq 可不填写 -app.white.list= diff --git a/TSG发布版本更新记录/TSG-21.12/hbase/update-hbase.txt b/TSG发布版本更新记录/TSG-21.12/hbase/update-hbase.txt index 91db508..0736f81 100644 --- a/TSG发布版本更新记录/TSG-21.12/hbase/update-hbase.txt +++ b/TSG发布版本更新记录/TSG-21.12/hbase/update-hbase.txt @@ -1,3 +1,10 @@ --- hbase新增表 +-- hbase新增表 在hbase服务器，使用hbase shell命令 -create 'tsg_galaxy:recommendation_app_cip', {NAME => 'common', VERSIONS => 1} \ No newline at end of file +create 'tsg_galaxy:recommendation_app_cip', {NAME => 'common', VERSIONS => 1} + + +-- hbase 新增字段 +disable "tsg_galaxy:job_result" +alter "tsg_galaxy:job_result",NAME=>'detail',TTL=>'1800' +alter "tsg_galaxy:job_result",NAME=>'result',TTL=>'1800' +enable "tsg_galaxy:job_result" diff --git a/TSG发布版本更新记录/TSG-21.12/hbase/update-phoenix.txt b/TSG发布版本更新记录/TSG-21.12/hbase/update-phoenix.txt index 7144a6f..d98eac5 100644 --- a/TSG发布版本更新记录/TSG-21.12/hbase/update-phoenix.txt +++ b/TSG发布版本更新记录/TSG-21.12/hbase/update-phoenix.txt @@ -1,11 +1,257 @@ + +#进入目录 phoenix-hbase-2.2-5.1.2-bin/bin 执行./sqlline.py 后分别执行以下建表语句 + -- phoenix 新增字段: alter view "tsg_galaxy"."relation_account_framedip" add "radius"."acct_status_type" UNSIGNED_INT; - -- Phoenix新增表 CREATE view "tsg_galaxy"."recommendation_app_cip"( ROWKEY VARCHAR PRIMARY KEY, "common"."app_label" VARCHAR, "common"."client_ip_list" VARCHAR, -"common"."last_update_time" UNSIGNED_LONG); \ No newline at end of file +"common"."last_update_time" UNSIGNED_LONG); + + +-- Phoenix新增表 +CREATE schema IF NOT EXISTS "tsg_galaxy"; +CREATE table IF NOT EXISTS "tsg_galaxy"."job_result"( +ROWKEY VARCHAR PRIMARY KEY, +"detail"."is_done" BOOLEAN, +"detail"."done_progress" UNSIGNED_FLOAT, +"detail"."is_canceled" BOOLEAN, +"detail"."discovery_field" VARCHAR, +"detail"."last_query_time" UNSIGNED_LONG, +"detail"."count" UNSIGNED_LONG, +"result"."app_extra_info" VARCHAR, +"result"."attack_type" VARCHAR, +"result"."bit_rate" VARCHAR, +"result"."common_action" VARCHAR, +"result"."common_address_type" VARCHAR, +"result"."common_app_label" VARCHAR, +"result"."common_c2s_byte_diff" VARCHAR, +"result"."common_c2s_byte_num" VARCHAR, +"result"."common_c2s_byte_retrans" VARCHAR, +"result"."common_c2s_ipfrag_num" VARCHAR, +"result"."common_c2s_pkt_diff" VARCHAR, +"result"."common_c2s_pkt_num" VARCHAR, +"result"."common_c2s_pkt_retrans" VARCHAR, +"result"."common_c2s_tcp_lostlen" VARCHAR, +"result"."common_c2s_tcp_unorder_num" VARCHAR, +"result"."common_client_asn" VARCHAR, +"result"."common_client_ip" VARCHAR, +"result"."common_client_location" VARCHAR, +"result"."common_client_port" VARCHAR, +"result"."common_con_duration_ms" VARCHAR, +"result"."common_data_center" VARCHAR, +"result"."common_device_group" VARCHAR, +"result"."common_device_id" VARCHAR, +"result"."common_direction" VARCHAR, +"result"."common_end_time" VARCHAR, +"result"."common_establish_latency_ms" VARCHAR, +"result"."common_external_ip" VARCHAR, +"result"."common_imei" VARCHAR, +"result"."common_imsi" VARCHAR, +"result"."common_internal_ip" VARCHAR, +"result"."common_l4_protocol" VARCHAR, +"result"."common_l7_protocol" VARCHAR, +"result"."common_mirrored_bytes" VARCHAR, +"result"."common_mirrored_pkts" VARCHAR, +"result"."common_phone_number" VARCHAR, +"result"."common_policy_id" VARCHAR, +"result"."common_protocol_label" VARCHAR, +"result"."common_s2c_byte_diff" VARCHAR, +"result"."common_s2c_byte_num" VARCHAR, +"result"."common_s2c_byte_retrans" VARCHAR, +"result"."common_s2c_ipfrag_num" VARCHAR, +"result"."common_s2c_pkt_diff" VARCHAR, +"result"."common_s2c_pkt_num" VARCHAR, +"result"."common_s2c_pkt_retrans" VARCHAR, +"result"."common_s2c_tcp_lostlen" VARCHAR, +"result"."common_s2c_tcp_unorder_num" VARCHAR, +"result"."common_schema_type" VARCHAR, +"result"."common_server_asn" VARCHAR, +"result"."common_server_ip" VARCHAR, +"result"."common_server_location" VARCHAR, +"result"."common_server_port" VARCHAR, +"result"."common_service_category" VARCHAR, +"result"."common_sessions" VARCHAR, +"result"."common_sled_ip" VARCHAR, +"result"."common_start_time" VARCHAR, +"result"."common_stream_dir" VARCHAR, +"result"."common_stream_error" VARCHAR, +"result"."common_stream_trace_id" VARCHAR, +"result"."common_sub_action" VARCHAR, +"result"."common_subscriber_id" VARCHAR, +"result"."common_tcp_client_isn" VARCHAR, +"result"."common_tcp_server_isn" VARCHAR, +"result"."common_tunnels" VARCHAR, +"result"."common_userdefine_app_name" VARCHAR, +"result"."conditions" VARCHAR, +"result"."destination_country" VARCHAR, +"result"."destination_ip" VARCHAR, +"result"."dns_aa" VARCHAR, +"result"."dns_ancount" VARCHAR, +"result"."dns_arcount" VARCHAR, +"result"."dns_message_id" VARCHAR, +"result"."dns_nscount" VARCHAR, +"result"."dns_opcode" VARCHAR, +"result"."dns_qclass" VARCHAR, +"result"."dns_qdcount" VARCHAR, +"result"."dns_qname" VARCHAR, +"result"."dns_qr" VARCHAR, +"result"."dns_qtype" VARCHAR, +"result"."dns_ra" VARCHAR, +"result"."dns_rcode" VARCHAR, +"result"."dns_rd" VARCHAR, +"result"."dns_sub" VARCHAR, +"result"."dns_tc" VARCHAR, +"result"."doh_aa" VARCHAR, +"result"."doh_ancount" VARCHAR, +"result"."doh_arcount" VARCHAR, +"result"."doh_cname" VARCHAR, +"result"."doh_cookie" VARCHAR, +"result"."doh_host" VARCHAR, +"result"."doh_message_id" VARCHAR, +"result"."doh_nscount" VARCHAR, +"result"."doh_opcode" VARCHAR, +"result"."doh_qclass" VARCHAR, +"result"."doh_qdcount" VARCHAR, +"result"."doh_qname" VARCHAR, +"result"."doh_qr" VARCHAR, +"result"."doh_qtype" VARCHAR, +"result"."doh_ra" VARCHAR, +"result"."doh_rcode" VARCHAR, +"result"."doh_rd" VARCHAR, +"result"."doh_referer" VARCHAR, +"result"."doh_rr" VARCHAR, +"result"."doh_sub" VARCHAR, +"result"."doh_tc" VARCHAR, +"result"."doh_url" VARCHAR, +"result"."doh_user_agent" VARCHAR, +"result"."doh_version" VARCHAR, +"result"."ftp_account" VARCHAR, +"result"."ftp_content" VARCHAR, +"result"."ftp_link_type" VARCHAR, +"result"."ftp_url" VARCHAR, +"result"."gtp_apn" VARCHAR, +"result"."gtp_downlink_teid" VARCHAR, +"result"."gtp_end_user_ipv4" VARCHAR, +"result"."gtp_end_user_ipv6" VARCHAR, +"result"."gtp_imei" VARCHAR, +"result"."gtp_imsi" VARCHAR, +"result"."gtp_msg_type" VARCHAR, +"result"."gtp_phone_number" VARCHAR, +"result"."gtp_uplink_teid" VARCHAR, +"result"."gtp_version" VARCHAR, +"result"."http_action_file_size" VARCHAR, +"result"."http_cookie" VARCHAR, +"result"."http_domain" VARCHAR, +"result"."http_host" VARCHAR, +"result"."http_referer" VARCHAR, +"result"."http_request_body" VARCHAR, +"result"."http_request_content_length" VARCHAR, +"result"."http_request_content_type" VARCHAR, +"result"."http_request_header" VARCHAR, +"result"."http_response_body" VARCHAR, +"result"."http_response_content_length" VARCHAR, +"result"."http_response_content_type" VARCHAR, +"result"."http_response_header" VARCHAR, +"result"."http_response_latency_ms" VARCHAR, +"result"."http_session_duration_ms" VARCHAR, +"result"."http_set_cookie" VARCHAR, +"result"."http_url" VARCHAR, +"result"."http_user_agent" VARCHAR, +"result"."http_version" VARCHAR, +"result"."mail_account" VARCHAR, +"result"."mail_attachment_name" VARCHAR, +"result"."mail_bcc" VARCHAR, +"result"."mail_cc" VARCHAR, +"result"."mail_eml_file" VARCHAR, +"result"."mail_from" VARCHAR, +"result"."mail_from_cmd" VARCHAR, +"result"."mail_protocol_type" VARCHAR, +"result"."mail_subject" VARCHAR, +"result"."mail_to" VARCHAR, +"result"."mail_to_cmd" VARCHAR, +"result"."packet_rate" VARCHAR, +"result"."quic_sni" VARCHAR, +"result"."quic_user_agent" VARCHAR, +"result"."quic_version" VARCHAR, +"result"."radius_account" VARCHAR, +"result"."radius_acct_authentic" VARCHAR, +"result"."radius_acct_delay_time" VARCHAR, +"result"."radius_acct_input_octets" VARCHAR, +"result"."radius_acct_input_packets" VARCHAR, +"result"."radius_acct_interim_interval" VARCHAR, +"result"."radius_acct_link_count" VARCHAR, +"result"."radius_acct_multi_session_id" VARCHAR, +"result"."radius_acct_output_octets" VARCHAR, +"result"."radius_acct_output_packets" VARCHAR, +"result"."radius_acct_session_id" VARCHAR, +"result"."radius_acct_session_time" VARCHAR, +"result"."radius_acct_status_type" VARCHAR, +"result"."radius_acct_terminate_cause" VARCHAR, +"result"."radius_callback_id" VARCHAR, +"result"."radius_callback_number" VARCHAR, +"result"."radius_called_station_id" VARCHAR, +"result"."radius_calling_station_id" VARCHAR, +"result"."radius_event_timestamp" VARCHAR, +"result"."radius_framed_ip" VARCHAR, +"result"."radius_framed_protocol" VARCHAR, +"result"."radius_idle_timeout" VARCHAR, +"result"."radius_nas_ip" VARCHAR, +"result"."radius_nas_port" VARCHAR, +"result"."radius_packet_type" VARCHAR, +"result"."radius_service_type" VARCHAR, +"result"."radius_session_timeout" VARCHAR, +"result"."radius_termination_action" VARCHAR, +"result"."rtp_originator_dir" VARCHAR, +"result"."rtp_payload_type_c2s" VARCHAR, +"result"."rtp_payload_type_s2c" VARCHAR, +"result"."rtp_pcap_path" VARCHAR, +"result"."session_rate" VARCHAR, +"result"."severity" VARCHAR, +"result"."sip_bye" VARCHAR, +"result"."sip_call_id" VARCHAR, +"result"."sip_duration" VARCHAR, +"result"."sip_originator_description" VARCHAR, +"result"."sip_originator_sdp_connect_ip" VARCHAR, +"result"."sip_originator_sdp_content" VARCHAR, +"result"."sip_originator_sdp_media_port" VARCHAR, +"result"."sip_originator_sdp_media_type" VARCHAR, +"result"."sip_responder_description" VARCHAR, +"result"."sip_responder_sdp_connect_ip" VARCHAR, +"result"."sip_responder_sdp_content" VARCHAR, +"result"."sip_responder_sdp_media_port" VARCHAR, +"result"."sip_responder_sdp_media_type" VARCHAR, +"result"."sip_server" VARCHAR, +"result"."sip_user_agent" VARCHAR, +"result"."source_country_list" VARCHAR, +"result"."source_ip_list" VARCHAR, +"result"."ssh_auth_success" VARCHAR, +"result"."ssh_cipher_alg" VARCHAR, +"result"."ssh_client_version" VARCHAR, +"result"."ssh_compression_alg" VARCHAR, +"result"."ssh_hassh" VARCHAR, +"result"."ssh_host_key" VARCHAR, +"result"."ssh_host_key_alg" VARCHAR, +"result"."ssh_kex_alg" VARCHAR, +"result"."ssh_mac_alg" VARCHAR, +"result"."ssh_server_version" VARCHAR, +"result"."ssh_version" VARCHAR, +"result"."ssl_cert_issuer" VARCHAR, +"result"."ssl_cert_subject" VARCHAR, +"result"."ssl_cert_verify" VARCHAR, +"result"."ssl_client_side_latency" VARCHAR, +"result"."ssl_client_side_version" VARCHAR, +"result"."ssl_cn" VARCHAR, +"result"."ssl_con_latency_ms" VARCHAR, +"result"."ssl_error" VARCHAR, +"result"."ssl_intercept_state" VARCHAR, +"result"."ssl_ja3_hash" VARCHAR, +"result"."ssl_pinningst" VARCHAR, +"result"."ssl_san" VARCHAR, +"result"."ssl_server_side_latency" VARCHAR, +"result"."ssl_server_side_version" VARCHAR, +"result"."ssl_sni" VARCHAR); \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/kafka/add-topic.txt b/TSG发布版本更新记录/TSG-21.12/kafka/add-topic.txt index 6529dd7..cdddf79 100644 --- a/TSG发布版本更新记录/TSG-21.12/kafka/add-topic.txt +++ b/TSG发布版本更新记录/TSG-21.12/kafka/add-topic.txt @@ -1,4 +1,4 @@ -#新增kafka topic +#新增kafka topic，在kafka服务器上运行以下命令，IP地址为zookeeper地址 分区副本以实际为准 kafka-topics.sh '--create' '--zookeeper' 'IP:2181/kafka' '--replication-factor' 1 '--partitions' 1 '--topic' TRAFFIC-APP-STAT \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/README.txt b/TSG发布版本更新记录/TSG-21.12/qgw/README.txt new file mode 100644 index 0000000..da68830 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/README.txt @@ -0,0 +1,6 @@ +更新配置文件步骤 + +1.修改backup_config.sh脚本中的配置后执行 +2.将备份中文件galaxy-qgw-service.yml复制一份放入更新目录galaxy-qgw-service/config/ +3.修改galaxy-qgw-service.yml增加更新文件update-galaxy-qgw-service.yml中的配置项 +4.修改push_config.sh 配置项后执行提交配置到nacos \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/backup_config.sh b/TSG发布版本更新记录/TSG-21.12/qgw/backup_config.sh new file mode 100644 index 0000000..d2a7fe0 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/backup_config.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +nacos_server="192.168.44.12" +tenant="prod" +group_id="Galaxy" +app_name="galaxy-qgw-service" +username="nacos" +password="nacos" + +#放置备份配置的目录 +backup_config_path=/home/tmp/21.11 + +search_config_url="http://$nacos_server:8848/nacos/v1/cs/configs?dataId=&group=${group_id}&appName=$app_name&username=$username&password=$password&tenant=$tenant&search=accurate&pageNo=1&pageSize=10000" + +backup_config_ids=$(curl -X GET $search_config_url|jq -r .pageItems[].dataId) + +array=(${backup_config_ids//,/ }) +if [ ! -d $backup_config_path ];then + mkdir -p $backup_config_path +fi + +for data_id in ${array[@]} +do + config_url="http://$nacos_server:8848/nacos/v1/cs/configs?dataId=$data_id&group=$group_id&appName=$app_name&username=$username&password=$password&tenant=$tenant" + + #备份 + curl $config_url > $backup_config_path/$data_id + +done + + #`curl -X DELETE $config_url` diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/active_defence_event.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/active_defence_event.json new file mode 100644 index 0000000..44aeb53 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/active_defence_event.json @@ -0,0 +1,322 @@ +{ + "type": "record", + "name": "active_defence_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "schema_query": { + "dimensions": [ + "common_policy_id", + "ad_target_ip", + "ad_cc_target_url" + ], + "metrics": [ + "ad_target_ip", + "ad_sent_byte_num", + "ad_sent_pkt_num", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num" + ], + "filters": [ + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_protocol", + "common_address_type", + "ad_sent_byte_num", + "ad_sent_pkt_num", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num" + ] + }, + "schema_type": { + "REFLECTION": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_target_ip", + "ad_target_port", + "ad_target_ip_location", + "ad_target_ip_asn", + "ad_reflector_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_reflector_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num" + ] + }, + "FLOOD": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_target_ip", + "ad_target_port", + "ad_target_ip_location", + "ad_target_ip_asn", + "ad_claimed_src_ip_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_claimed_src_ip_profile_id", + "ad_protocol" + ] + }, + "CC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_cc_target_url", + "ad_claimed_src_ip_profile_id", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_cc_target_url", + "ad_claimed_src_ip_profile_id", + "ad_protocol" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_cc_target_url" + ] + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ad_target_ip", + "label": "Target IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_ip_country,geo_asn", + "appendTo": "ad_target_ip_location,ad_target_ip_asn" + } + }, + "type": "string" + }, + { + "name": "ad_target_port", + "label": "Target Port", + "type": "int" + }, + { + "name": "ad_cc_target_url", + "label": "Target URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ad_target_ip_location", + "label": "Target Location", + "type": "string" + }, + { + "name": "ad_target_ip_asn", + "label": "Target ASN", + "type": "string" + }, + { + "name": "ad_protocol", + "label": "Protocol", + "type": "string" + }, + { + "name": "ad_method", + "label": "Method", + "type": "string" + }, + { + "name": "ad_claimed_src_ip_profile_id", + "label": "Claimed Profile ID", + "type": "int" + }, + { + "name": "ad_reflector_profile_id", + "label": "Reflector Profile ID", + "type": "int" + }, + { + "name": "ad_sent_pkt_num", + "label": "Packets Sent", + "type": "int" + }, + { + "name": "ad_sent_byte_num", + "label": "Bytes Sent", + "type": "int" + }, + { + "name": "ad_cc_initiate_connection_num", + "label": "Initiate Numbers", + "type": "int" + }, + { + "name": "ad_cc_established_connection_num", + "label": "Established Numbers", + "type": "int" + }, + { + "name": "ad_cc_rejected_connection_num", + "label": "Rejected Numbers", + "type": "int" + }, + { + "name": "ad_generate_time", + "label": "Generate Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "int" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-filter.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-filter.json new file mode 100644 index 0000000..a0a03b0 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-filter.json @@ -0,0 +1,99 @@ +{ + "version": "1.0", + "name": "ClickHouse-Raw", + "namespace": "ClickHouse", + "filters": [ + { + "name":"@start", + "value": "'2021-10-19 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-10-20 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='118.180.48.74'", + "common_client_ip='120.242.132.200'", + "common_internal_ip='223.116.37.192'", + "common_server_ip='8.8.8.8'", + "common_server_ip='114.114.114.114'", + "common_server_ip!='114.114.114.114'", + "common_server_ip='120.239.72.226'", + "common_external_ip='111.10.53.14'", + "common_client_port=52607", + "common_server_port=443", + "common_c2s_pkt_num>5", + "common_s2c_pkt_num>5", + "common_c2s_byte_num>100", + "common_s2c_byte_num<200", + "common_schema_type='DNS'", + "common_establish_latency_ms>200", + "common_con_duration_ms>10000", + "common_stream_trace_id=1153021139190754263", + "common_tcp_client_isn=2857077935", + "common_tcp_server_isn=0", + "http_domain='qq.com'", + "http_domain!='qq.com'", + "http_domain='yunser.com'", + "mail_account='abc@xx.com'", + "mail_subject='test'", + "dns_qname='qbwup.imtt.qq.com'", + "ssl_sni='mmbiz.qpic.cn'", + "ssl_sni='openai.qq.com'", + "ssl_con_latency_ms>100", + "ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'", + "common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'", + "common_server_ip='111.10.53.14' and common_server_port=443", + "common_server_ip like '120.239%'", + "common_server_ip not like '120.239%'", + "common_server_ip like '%114.114%'", + "mail_account like 'abc@%'", + "http_domain like '%baidu.com%'", + "ssl_sni like '%google.com'", + "http_domain like 'baidu%'", + "http_domain like '%baidu.com%'", + "common_client_ip in ('120.239.72.226','114.114.114.114')", + "common_client_ip not in ('120.239.72.226','114.114.114.114')", + "common_server_ip='116.177.248.126' and notEmpty(http_domain)", + "common_server_ip='116.177.248.126' and common_client_ip='120.242.132.200'", + "common_server_ip='116.177.248.126' and common_stream_trace_id=1153021139190754263", + "common_client_ip='120.242.132.200' and common_server_ip='116.177.248.126'", + "http_domain='qq.com' or common_server_ip='120.239.72.226'", + "common_server_port not in (80,443)", + "http_domain not like '%qq.com'" + ] + }, + { + "name":"@index_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='118.180.48.74'", + "common_client_ip='120.242.132.200'", + "common_server_ip='114.114.114.114'", + "common_server_ip!='114.114.114.114'", + "common_server_ip='120.239.72.226'", + "http_domain='qq.com'", + "http_domain!='qq.com'", + "http_domain='yunser.com'", + "ssl_sni='mmbiz.qpic.cn'", + "ssl_sni='openai.qq.com'", + "common_server_ip like '120.239%'", + "common_server_ip not like '120.239%'", + "common_server_ip like '%114.114%'", + "common_subscriber_id='%test%'", + "http_domain like 'baidu%'", + "http_domain like '%baidu.com%'", + "common_client_ip in ('120.239.72.226','114.114.114.114')", + "common_client_ip not in ('120.239.72.226','114.114.114.114')", + "common_server_ip='116.177.248.126' and notEmpty(http_domain)", + "common_server_ip='116.177.248.126' and common_client_ip='120.242.132.200'", + "common_server_ip='116.177.248.126' and common_stream_trace_id=1153021139190754263", + "common_client_ip='120.242.132.200' and common_server_ip='116.177.248.126'", + "http_domain='qq.com' or common_server_ip='120.239.72.226'" + ] + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-queries-template.sql b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-queries-template.sql new file mode 100644 index 0000000..fc08e8c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/ck-queries-template.sql @@ -0,0 +1,118 @@ +--Q01.Count(1) +select count(1) from session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) +--Q02.All Fields Query (default) +SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) LIMIT 30 +--Q03.All Fields Query order by Time desc +SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q04.All Fields Query order by Time asc +SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time asc LIMIT 30 +--Q05.All Fields Query by Filter +SELECT * FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30 +--Q06.Default Fields Query by Filter +SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30 +--Q07.All Fields Query (sub query by time) +SELECT * FROM session_record AS session_record WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q08.All Fields Query (sub query by log id) +SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q09.Default Field Query (sub query by time) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q10.Default Field Query (sub query by log id) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( select common_log_id FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)) ORDER BY common_recv_time DESC LIMIT 30 +--Q11.Default Field Query by Server IP (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_common_server_ip AS session_record_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q12.Default Field Query by Client IP (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_common_client_ip AS session_record_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q13.Default Field Query by Domain (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM session_record AS session_record WHERE common_log_id IN ( Select common_log_id FROM session_record_http_domain AS session_record_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q14.All Fields Query by Client IP (sub query by log id with index Table) +SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_common_client_ip AS session_record_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q15.All Fields Query by Server IP(sub query by log id with index Table) +SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_common_server_ip AS session_record_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q16.All Fields Query by Domain(sub query by log id with index Table) +SELECT * FROM session_record AS session_record WHERE common_log_id IN ( SELECT common_log_id FROM session_record_http_domain AS session_record_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q17.Session Logs Sent to Database Trend(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", count(common_log_id) AS "logs" FROM session_record AS session_record WHERE ( ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q18.Traffic Bandwidth Trend(Time Grain 30 second) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 30 SECOND)))) AS stat_time, sum(common_c2s_byte_num) AS bytes_sent, sum(common_s2c_byte_num) AS bytes_received, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets, sum(common_sessions) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time ORDER BY stat_time ASC LIMIT 10000 +--Q19.Log Tend by Type (Time Grain 5 minute) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) GROUP BY stat_time, common_schema_type ORDER BY stat_time ASC LIMIT 10000 +--Q20.Traffic Metrics Analytic +SELECT round(sum(common_s2c_byte_num) * 8 / 300,2) AS trafficInBits, round(sum(common_c2s_byte_num) * 8 / 300,2) AS trafficOutBits, round(sum(common_s2c_byte_num + common_c2s_byte_num) * 8 / 300,2) AS trafficTotalBits, round(sum(common_s2c_pkt_num) / 300,2) AS trafficInPackets, round(sum(common_c2s_pkt_num) / 300,2) AS trafficOutPackets, round(sum(common_s2c_pkt_num + common_c2s_pkt_num) / 300,2) AS trafficTotalPackets, round(sum(common_sessions) / 300,2) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) +--Q21.Traffic Endpoints Metrics Trend(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", uniq(common_internal_ip) AS "Unique Internal IP", uniq(common_external_ip) AS "Unique External IP", uniq(common_subscriber_id) AS "Unique Subscriber ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q22.Endpoint Unique Num by L4 Protocol +SELECT 'all' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) UNION ALL SELECT 'tcp' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) UNION ALL SELECT 'UDP' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_UDP', 'IPv6_UDP' ) +--Q23.One-sided Connection Trend(Time Grain 5 minute) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, (CASE WHEN common_stream_dir = 1 THEN 'c2s' WHEN common_stream_dir = 2 THEN 's2c' WHEN common_stream_dir = 3 THEN 'double' ELSE 'None' END) AS type, sum(common_sessions) AS sessions FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time, common_stream_dir ORDER BY stat_time ASC LIMIT 10000 +--Q24. Estimated One-sided Sessions with Bandwidth +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_sessions) AS "sessions", sum(if(common_stream_dir <> 3, common_sessions, 0)) AS "one_side_sessions", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", round(one_side_sessions / sessions, 2) AS one_side_percent FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q25.Estimated TCP Sequence Gap Loss +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", sum(common_c2s_tcp_lostlen + common_s2c_tcp_lostlen) AS "gap_loss_bytes", round(gap_loss_bytes / bytes, 2) AS gap_loss_percent FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q26.Top30 Server IP by Bytes +SELECT "server_ip" AS "server_ip" , SUM(coalesce("bytes",0)) AS "bytes" , SUM(coalesce("bytes_sent",0)) AS "Sent" , SUM(coalesce("bytes_received",0)) AS "Received" , SUM(coalesce("sessions",0)) AS "sessions" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(common_c2s_byte_num+common_s2c_byte_num) AS "bytes" , SUM(coalesce(common_sessions,0)) AS "sessions" , common_server_ip AS "server_ip" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" desc ) GROUP BY "server_ip" ORDER BY "bytes" desc LIMIT 30 +--Q27.Top30 Client IP by Sessions +SELECT common_client_ip , COUNT(*) AS sessions FROM session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_client_ip ORDER BY sessions desc LIMIT 0,30 +--Q28.Top30 TCP Server Ports by Sessions +SELECT "Server Port" AS "Server Port", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_port AS "Server Port", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Server Port" LIMIT 1048576) GROUP BY "Server Port" ORDER BY "Sessions" DESC LIMIT 30 +--Q29.Top30 Domian by Bytes +SELECT "domain" AS "Website Domain" , SUM(coalesce("bytes",0)) AS "Throughput" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "bytes" , http_domain AS "domain" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "domain" ORDER BY "bytes" desc ) GROUP BY "domain" ORDER BY "Throughput" desc LIMIT 30 +--Q30.Top30 Endpoint Devices by Bandwidth +SELECT "device_id" AS "device_id", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, common_device_id AS "device_id" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 30 +--Q31.Top30 Domain by Unique Client IP +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Client IP", 0)) AS "Client IP" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Client IP" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 30 +--Q32.Top100 Most Time Consuming Domains +SELECT "Domain" AS "Domain", avg(coalesce("Avg Establish Latency(ms)", 0)) AS "Avg Establish Latency(ms)" FROM (SELECT http_domain AS "Domain", avg(coalesce(common_establish_latency_ms, 0)) AS "Avg Establish Latency(ms)" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Avg Establish Latency(ms)" DESC LIMIT 100 +--Q33.Top30 Sources by Sessions +SELECT "source" AS "source", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(common_subscriber_id, ''), nullif(common_client_ip, '')) AS "source", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 30 +--Q34.Top30 Destinations by Sessions +SELECT "destination" AS "destination", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(http_domain, ''), nullif(common_server_ip, '')) AS "destination", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 30 +--Q35.Top30 Destination Regions by Bandwidth +SELECT "server_location" AS "server_location", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT arrayElement(splitByString(',', common_server_location), length(splitByString(',', common_server_location))) AS "server_location", sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "bytes", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 30 +--Q36.Top30 URLS by Sessions +SELECT "Http URL" AS "Http URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "Http URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http URL" LIMIT 1048576) GROUP BY "Http URL" ORDER BY "Sessions" DESC LIMIT 30 +--Q37.Top30 Destination Transmission APP by Bandwidth +SELECT "server_ip" AS "server_ip", groupUniqArray(coalesce("trans_app", 0)) AS "trans_app", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", groupUniqArray(concat(common_l4_protocol, '/', toString(common_server_port))) AS "trans_app", common_server_ip AS "server_ip" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 30 +--Q38.Browsing Users by Website domains and Sessions +SELECT "Subscriber ID" AS "Subscriber ID", "Http.Domain" AS "Http.Domain", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT http_domain AS "Http.Domain", common_subscriber_id AS "Subscriber ID", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) AND notEmpty(common_subscriber_id) ) GROUP BY "Http.Domain", "Subscriber ID" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Subscriber ID", "Http.Domain" ORDER BY "sessions" DESC LIMIT 10000 +--Q39.Top Domain and Server IP by Bytes Sent +SELECT "Http.Domain" AS "Http.Domain" , "Server IP" AS "Server IP" , SUM(coalesce("Bytes Sent",0)) AS "Bytes Sent" FROM ( SELECT common_server_ip AS "Server IP" , http_domain AS "Http.Domain" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "Bytes" , SUM(coalesce(common_c2s_byte_num,0)) AS "Bytes Sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "Bytes Received" FROM session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "Server IP" , "Http.Domain" ORDER BY "Bytes" desc LIMIT 1048576 ) GROUP BY "Http.Domain" , "Server IP" ORDER BY "Bytes Sent" desc LIMIT 10000 +--Q40.Top30 Website Domains by Client IP and Sessions +SELECT "Http.Domain" AS "Http.Domain", "Client IP" AS "Client IP", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT common_client_ip AS "Client IP", http_domain AS "Http.Domain", sum(coalesce(common_sessions, 0)) AS "sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Client IP", "Http.Domain" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Http.Domain", "Client IP" ORDER BY "sessions" DESC LIMIT 10000 +--Q41.Domain is Accessed by Unique Client IP Trend(bytes Time Grain 5 minute) +SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , http_domain AS Domain, COUNT(DISTINCT(common_client_ip)) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) AND http_domain IN ( SELECT http_domain FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY SUM(common_s2c_byte_num+common_c2s_byte_num) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , http_domain ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000 +--Q42. Domain is Accessed by Unique Client IP Trend(sessions,Time Grain 5 minute) +SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),3600)*3600) AS stat_time , http_domain , uniq (common_client_ip) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start)-604800 AND common_recv_time < toDateTime(@end) AND http_domain IN ( SELECT http_domain FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY COUNT(*) desc LIMIT 5 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 3600)*3600), http_domain ORDER BY stat_time desc LIMIT 10000 +--Q43.Bandwidth Trend with Device ID(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", common_device_id AS "Device ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time", "Device ID" LIMIT 10000 +--Q44.Internal IP by Sled IP and Sessions +SELECT "Internal IP" AS "Internal IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_internal_ip AS "Internal IP", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Sled IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 10000 +--Q45.Bandwidth Trend with Internal IP (Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_c2s_pkt_num + common_s2c_pkt_num, 0)) AS "Packets", sum(coalesce(common_sessions, 0)) AS "New Sessions", sum(coalesce(common_c2s_byte_num, 0)) AS "Bytes Sent", sum(coalesce(common_s2c_byte_num, 0)) AS "Bytes Received", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent", sum(coalesce(common_s2c_pkt_num, 0)) AS "Packets Received" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) GROUP BY "Receive Time" LIMIT 10000 +--Q46.Top30 Domains Detail with Internal IP +SELECT "Domain" AS "Domain", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_domain AS "Domain", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Sessions" DESC LIMIT 30 +--Q47.Top30 URLS Detail with Internal IP +SELECT "URL" AS "URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_url) ) GROUP BY "URL" LIMIT 1048576) GROUP BY "URL" ORDER BY "Sessions" DESC LIMIT 30 +--Q48.Top Domains with Unique Client IP and Subscriber ID +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Unique Client IP", 0)) AS "Unique Client IP", sum(coalesce("Unique Subscriber ID", 0)) AS "Unique Subscriber ID" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Unique Client IP", uniq(common_subscriber_id) AS "Unique Subscriber ID" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Unique Client IP" DESC LIMIT 100 +--Q49.Top100 Domains by Packets sent +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Packets Sent", 0)) AS "Packets Sent" FROM (SELECT http_domain AS "Http.Domain", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Packets Sent" DESC LIMIT 100 +--Q50.Internal and External asymmetric traffic +SELECT "Internal IP" AS "Internal IP", "External IP" AS "External IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_external_ip AS "External IP", common_internal_ip AS "Internal IP", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes Sent+Bytes Received", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Sled IP", "External IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "External IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 500 +--Q51.Client and Server ASN asymmetric traffic +SELECT "Client ASN" AS "Client ASN", "Server ASN" AS "Server ASN", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_asn AS "Server ASN", common_client_asn AS "Client ASN", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Server ASN", "Client ASN" LIMIT 1048576) GROUP BY "Client ASN", "Server ASN" ORDER BY "Sessions" DESC LIMIT 500 +--Q52.Top handshake latency by Website and Client IPs +SELECT "SSL.SNI" AS "SSL.SNI", "Client IP" AS "Client IP", avg(coalesce("Establish Latency(ms)", 0)) AS "Establish Latency(ms)" FROM (SELECT common_client_ip AS "Client IP", ssl_sni AS "SSL.SNI", avg(coalesce(common_establish_latency_ms, 0)) AS "Establish Latency(ms)" FROM session_record AS session_record WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Client IP", "SSL.SNI" LIMIT 1048576) GROUP BY "SSL.SNI", "Client IP" ORDER BY "Establish Latency(ms)" DESC LIMIT 500 +--Q53.Domain baidu.com Drill down Client IP +select common_client_ip as "Client IP" , avg(common_establish_latency_ms) as "Establishing Time Mean(ms)", count(1) as Responses,any(common_client_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Client IP" order by Responses desc limit 100 +--Q54.Domain baidu.com Drill down Server IP +select common_server_ip as "Server IP" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses,any(common_server_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Server IP" order by Responses desc limit 100 +--Q55.Domain baidu.com Drill down URI +select http_url as "URI" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "URI" order by Responses desc limit 100 +--Q56.L7 Protocol Metrics +select common_l7_protocol as "Protocol" , uniq(common_client_ip) as "Clients" , uniq(common_server_ip) as "Servers", count(1) as Sessions,sum(common_c2s_byte_num+common_s2c_byte_num) as bytes from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and notEmpty(common_l7_protocol) group by common_l7_protocol order by bytes desc +--Q57.L7 Protocol SIP Drill down Client IP +select common_client_ip as "Client IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_client_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Client IP" order by Sessions desc limit 100 +--Q58.L7 Protocol SIP Drill down Server IP +select common_server_ip as "Server IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_server_location) as Location from session_record where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Server IP" order by Sessions desc limit 100 +--Q59.Top5 Server IP keys with Unique Client IPs Trend (Grain 5 minute) +SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , common_server_ip AS server_ip, COUNT(DISTINCT(common_client_ip)) AS nums FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_server_ip IN ( SELECT common_server_ip FROM session_record AS session_record WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_server_ip ORDER BY count(*) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , server_ip ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000 \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/columns_cluster.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/columns_cluster.json new file mode 100644 index 0000000..d190d3c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/columns_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "columns_cluster", + "fields": [ + { + "name": "database", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/disks_cluster.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/disks_cluster.json new file mode 100644 index 0000000..70777c6 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/disks_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "disks_cluster", + "fields": [ + { + "name": "name", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/dos_event.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/dos_event.json similarity index 98% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/dos_event.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/dos_event.json index c17a729..9d34fb6 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/dos_event.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/dos_event.json @@ -187,9 +187,9 @@ "session_rate" ], "internal_columns": [ - "common_recv_time", - "common_log_id", - "common_processing_time" + "start_time", + "log_id", + "end_time" ] }, "fields": [ diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-filter.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-filter.json new file mode 100644 index 0000000..e8286b7 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-filter.json @@ -0,0 +1,21 @@ +{ + "version": "1.0", + "name": "druid-Raw", + "namespace": "druid", + "filters": [ + { + "name":"@start", + "value": "'2021-10-19 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-10-20 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_client_ip='192.168.44.21'and common_server_port=443" + ] + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-queries-template.sql b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-queries-template.sql new file mode 100644 index 0000000..c56d2c8 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/druid-queries-template.sql @@ -0,0 +1,92 @@ +--Q01.All Security Event Hits +select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end group by policy_id +--Q02.Security Event Hits with Policy ID 0 +select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id in (0) group by policy_id +--Q03.All Security Event Hits Trend by 5min A +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from security_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 10000 +--Q04.Security Event Hit Time（first and last time) A +select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from security_event_hits_log where policy_id in (0) group by policy_id +--Q05.Top 200 Security Policies +select policy_id, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP @start and __time =@start and __time <@end group by policy_id, action order by hits desc limit 200 +--Q07.All Proxy Event Hits +select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id +--Q08.Proxy Event Hits with Policy ID 0 +select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=0 group by policy_id +--Q09.All Proxy Event Hits Trend by 5min A +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time =TIMESTAMP @start and __time =@start and __time <@end group by policy_id, sub_action order by hits desc limit 200 +--Q13.Proxy Action Hits +select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by sub_action +--Q14.Proxy Action Hits Trend by 5min +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time = @start AND __time < @end UNION ALL SELECT sum(pinning_num) AS sessions, 'pinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(maybe_pinning_num) AS sessions, 'maybePinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end +--Q16.Traffic Metrics Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q17.Traffic Metrics Not Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(not_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time>= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q18.Traffic Metrics Maybe Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(maybe_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q19.Traffic Metrics Throughput Bytes IN/OUT +select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q23.Traffic Metrics Bandwidth Packets IN/OUT +select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q25.Traffic Metrics New and Live Sessions +select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q27.Traffic Metrics Security Throughput Bytes +select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end +--Q28.Traffic Metrics Security Throughput Packets +select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time = TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') +--Q31.Traffic Metrics Security Bandwidth Packets by 5Min +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100 +--Q35.Top 100 Internal IP by Sessions +select source as internal_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >=TIMESTAMP @start and __time = @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100 +--Q37.Top 100 Domain by Bytes +select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >=TIMESTAMP @start and __time =TIMESTAMP @start and __time =TIMESTAMP @start and __time = @start and __time < @end group by policy_id +--Q41.Traffic Composition Metrics +SELECT APPROX_COUNT_DISTINCT_DS_HLL(ip_object) AS uniq_client_ip, SUM(one_sided_connections) AS one_sided_connections, SUM(uncategorized_bytes) AS total_uncategorized_bytes, SUM(fragmentation_packets) AS fragmentation_packets, SUM(sequence_gap_loss) AS sequence_gap_loss_bytes, SUM(s2c_byte_num+c2s_byte_num) AS summaryTotalBytes, SUM(s2c_pkt_num+c2s_pkt_num) AS summaryTotalPackets, SUM(sessions) AS summarySessions FROM traffic_summary_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end LIMIT 1 +--Q42.Traffic Composition Throughput +(SELECT SUM(c2s_byte_num + s2c_byte_num) as total_bytes, SUM(sessions) as total_sessions, (SUM(c2s_byte_num + s2c_byte_num) * 8)/((TIMESTAMP_TO_MILLIS(TIMESTAMP @end )-TIMESTAMP_TO_MILLIS(TIMESTAMP @start ))/1000) AS data_rate FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' LIMIT 1) UNION ALL ( SELECT SUM(sessions), 0, 0 FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' GROUP BY __time ORDER BY __time DESC LIMIT 1 ) +--Q43.Traffic Composition Protocol Tree +SELECT protocol_id, SUM(sessions) as sessions,SUM(c2s_byte_num) as c2s_byte_num, SUM(c2s_pkt_num) as c2s_pkt_num, SUM(s2c_byte_num) as s2c_byte_num, SUM(s2c_pkt_num) as s2c_pkt_num FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end GROUP BY protocol_id +--Q44.System Quota +SELECT log_type, SUM(used_size) as used_size, SUM(max_size) * 7/10 as max_size, TIME_FORMAT(LATEST(last_storage) * 1000,'YYYY-MM-dd') as first_storage FROM ( SELECT log_type, LATEST(used_size) as used_size, LATEST(max_size) as max_size, LATEST(last_storage) as last_storage FROM sys_storage_log WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '1' HOUR AND data_center != '' GROUP BY data_center,log_type ) GROUP BY log_type +--Q45.System Quota Daily Trend +select TIME_FORMAT(__time,'YYYY-MM-dd') as stat_time,log_type as type, sum(aggregate_size) as used_size from sys_storage_log where __time >= @start and __time < @end group by TIME_FORMAT(__time,'YYYY-MM-dd'), log_type +--Q46.Traffic Statistics(Metrics01) +select sum(total_hit_sessions) as total_hit_sessions, sum(total_bytes_transferred) as total_bytes_transferred, sum(total_packets_transferred) as total_packets_transferred, sum(total_new_sessions) as total_new_sessions , sum(total_close_sessions) as total_close_sessions, sum(average_new_sessions_per_second) as average_new_sessions_per_second , sum(average_bytes_per_second) as average_bytes_per_second , sum(average_packets_per_second) as average_packets_per_second , COUNT(DISTINCT(device_id)) as device_num, sum(live_sessions) as average_live_sessions from ( select device_id, sum(intercept_conn_num + monitor_conn_num + deny_conn_num + allow_conn_num) as total_hit_sessions, sum(total_in_bytes + total_out_bytes) as total_bytes_transferred, sum(total_in_packets + total_out_packets) as total_packets_transferred, sum(new_conn_num) as total_new_sessions, sum(close_conn_num) as total_close_sessions, avg(nullif(new_conn_num, 0))/ 5 as average_new_sessions_per_second, avg(nullif(total_in_bytes + total_out_bytes, 0))* 8 / 5 as average_bytes_per_second, avg(nullif(total_in_packets + total_out_packets, 0))/ 5 as average_packets_per_second, avg(nullif(established_conn_num, 0)) as live_sessions from traffic_metrics_log where __time >= @start and __time < @end group by device_id) \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/engine-filter.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/engine-filter.json new file mode 100644 index 0000000..525a02c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/engine-filter.json @@ -0,0 +1,53 @@ +{ + "version": "1.0", + "name": "Engine-Raw", + "namespace": "Engine", + "filters": [ + { + "name":"@start", + "value": "'2021-10-19 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-10-20 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='36.189.226.21'", + "common_internal_ip='223.116.37.192'", + "common_server_ip='8.8.8.8'", + "common_external_ip='111.10.53.14'", + "common_client_port=52607", + "common_server_port=443", + "common_c2s_pkt_num>5", + "common_s2c_pkt_num>5", + "common_c2s_byte_num>100", + "common_s2c_byte_num<200", + "common_schema_type='DNS'", + "common_establish_latency_ms>200", + "common_con_duration_ms>10000", + "common_stream_trace_id=1153021139190754263", + "common_tcp_client_isn=2857077935", + "common_tcp_server_isn=0", + "http_domain='microsoft.com'", + "mail_account='abc@xx.com'", + "mail_subject='test'", + "dns_qname='qbwup.imtt.qq.com'", + "ssl_sni='note.youdao.com'", + "ssl_con_latency_ms>100", + "ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'", + "common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'", + "common_server_ip='111.10.53.14' and common_server_port=443", + "mail_account like 'abc@%'", + "http_domain like '%baidu.com%'", + "ssl_sni like '%youdao.com'", + "common_client_ip in ('36.189.226.21','111.10.53.14')", + "common_server_port not in (80,443)", + "notEmpty(http_domain)", + "http_domain not like '%microsoft.com'" + ] + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/engine-queries-template.sql b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/engine-queries-template.sql similarity index 100% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/engine-queries-template.sql rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/engine-queries-template.sql diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/gtpc_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/gtpc_record.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/gtpc_record.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/gtpc_record.json index f2164f7..cfa34f5 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/gtpc_record.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/gtpc_record.json @@ -780,7 +780,7 @@ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, - "allow_query": "true" + "visibility": "hidden" }, "type": "string" }, diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-filter.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/hbase-filter.json similarity index 100% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-filter.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/hbase-filter.json diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-queries-template.sql b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/hbase-queries-template.sql similarity index 100% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/hbase-queries-template.sql rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/hbase-queries-template.sql diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/interim_session_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/interim_session_record.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/interim_session_record.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/interim_session_record.json index edf29c3..01f5c87 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/interim_session_record.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/interim_session_record.json @@ -263,7 +263,9 @@ "internal_columns": [ "common_recv_time", "common_log_id", - "common_processing_time" + "common_processing_time", + "common_userdefine_app_name", + "common_tunnels" ], "tunnel_type": { "$ref": "public_schema_info.json#/tunnel_type" @@ -747,7 +749,7 @@ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, - "allow_query": "true" + "visibility": "hidden" }, "type": "string" }, diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/job_result.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/job_result.json new file mode 100644 index 0000000..6478cb4 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/job_result.json @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "job_result", + "namespace": "tsg_galaxy", + "fields": [ + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_interim.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_interim.json new file mode 100644 index 0000000..0898ce1 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_interim.json @@ -0,0 +1,169 @@ +{ + "type": "record", + "name": "liveChart_interim", + "in": "INTERIM-SESSION-RECORD", + "out": "TRAFFIC-PROTOCOL-STAT", + "task": "Protocol-Distribution", + "doc": { + "timestamp": { + "name": "stat_time", + "type": "long" + }, + "dimensions": [ + { + "name": "protocol_id", + "fieldName": "common_protocol_label", + "type": "string" + }, + { + "name": "entrance_id", + "fieldName": "common_entrance_id", + "type": "string" + }, + { + "name": "isp", + "fieldName": "common_isp", + "type": "string" + }, + { + "name": "data_center", + "fieldName": "common_data_center", + "type": "string" + }, + { + "name": "device_group", + "fieldName": "common_device_group", + "type": "string" + } + ], + "metrics": [ + { + "function": "sum", + "name": "sessions", + "fieldName": "common_sessions", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_byte_num", + "fieldName": "common_c2s_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_byte_num", + "fieldName": "common_s2c_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_pkt_num", + "fieldName": "common_c2s_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_pkt_num", + "fieldName": "common_s2c_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_ipfrag_num", + "fieldName": "common_c2s_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_ipfrag_num", + "fieldName": "common_s2c_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_lostlen", + "fieldName": "common_c2s_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_lostlen", + "fieldName": "common_s2c_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_unorder_num", + "fieldName": "common_c2s_tcp_unorder_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_unorder_num", + "fieldName": "common_s2c_tcp_unorder_num", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_sip_num", + "fieldName": "common_server_ip", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_cip_num", + "fieldName": "common_client_ip", + "type": "long" + } + ], + "filters": [ + { + "fieldName": "common_protocol_label", + "type": "notempty" + } + ], + "transforms": [ + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_l7_protocol,." + }, + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_app_label,." + }, + { + "function": "flattenSpec", + "name": "data_center", + "fieldName": "common_device_tag", + "parameters": "$.tags[?(@.tag=='data_center')].value" + }, + { + "function": "flattenSpec", + "name": "device_group", + "fieldName": "common_device_tag", + "parameters": "$.tags[?(@.tag=='device_group')].value" + }, + { + "function": "hierarchy", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "." + } + ], + "action": [ + { + "label": "Default", + "metrics": "c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num" + } + ], + "granularity": { + "type": "period", + "period": "15S" + } + }, + "fields": [] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_session.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_session.json new file mode 100644 index 0000000..8a2c499 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/liveChart_session.json @@ -0,0 +1,169 @@ +{ + "type": "record", + "name": "liveChart_session", + "in": "SESSION-RECORD", + "out": "TRAFFIC-PROTOCOL-STAT", + "task": "Protocol-Distribution", + "doc": { + "timestamp": { + "name": "stat_time", + "type": "long" + }, + "dimensions": [ + { + "name": "protocol_id", + "fieldName": "common_protocol_label", + "type": "string" + }, + { + "name": "entrance_id", + "fieldName": "common_entrance_id", + "type": "string" + }, + { + "name": "isp", + "fieldName": "common_isp", + "type": "string" + }, + { + "name": "data_center", + "fieldName": "common_data_center", + "type": "string" + }, + { + "name": "device_group", + "fieldName": "common_device_group", + "type": "string" + } + ], + "metrics": [ + { + "function": "sum", + "name": "sessions", + "fieldName": "common_sessions", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_byte_num", + "fieldName": "common_c2s_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_byte_num", + "fieldName": "common_s2c_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_pkt_num", + "fieldName": "common_c2s_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_pkt_num", + "fieldName": "common_s2c_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_ipfrag_num", + "fieldName": "common_c2s_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_ipfrag_num", + "fieldName": "common_s2c_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_lostlen", + "fieldName": "common_c2s_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_lostlen", + "fieldName": "common_s2c_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_unorder_num", + "fieldName": "common_c2s_tcp_unorder_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_unorder_num", + "fieldName": "common_s2c_tcp_unorder_num", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_sip_num", + "fieldName": "common_server_ip", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_cip_num", + "fieldName": "common_client_ip", + "type": "long" + } + ], + "filters": [ + { + "fieldName": "common_protocol_label", + "type": "notempty" + } + ], + "transforms": [ + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_l7_protocol,." + }, + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_app_label,." + }, + { + "function": "flattenSpec", + "name": "data_center", + "fieldName": "common_device_tag", + "parameters": "$.tags[?(@.tag=='data_center')].value" + }, + { + "function": "flattenSpec", + "name": "device_group", + "fieldName": "common_device_tag", + "parameters": "$.tags[?(@.tag=='device_group')].value" + }, + { + "function": "hierarchy", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "." + } + ], + "action": [ + { + "label": "Default", + "metrics": "sessions,c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num,c2s_ipfrag_num,s2c_ipfrag_num,c2s_tcp_lostlen,s2c_tcp_lostlen,c2s_tcp_unorder_num,s2c_tcp_unorder_num" + } + ], + "granularity": { + "type": "period", + "period": "15S" + } + }, + "fields": [] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/meta_data.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/meta_data.json similarity index 97% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/meta_data.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/meta_data.json index 23f1f5f..3003273 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/meta_data.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/meta_data.json @@ -81,8 +81,8 @@ "group": "HBASE_GROUP", "tables": [ "relation_account_framedip", - "job_result", - "recommendation_app_cip" + "recommendation_app_cip", + "job_result" ] } ] diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/parts_cluster.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/parts_cluster.json new file mode 100644 index 0000000..c311abf --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/parts_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "parts_cluster", + "fields": [ + { + "name": "name", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/processes.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/processes.json new file mode 100644 index 0000000..75d74a9 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/processes.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "processes", + "fields": [ + { + "name": "query_id", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/proxy_event.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/proxy_event.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event.json index acd9e73..6d9c1c4 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/proxy_event.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event.json @@ -633,6 +633,10 @@ { "code": "hijack", "value": "Hijack" + }, + { + "code": "edit_element", + "value": "Edit Element" } ], "allow_query": "true" @@ -701,7 +705,7 @@ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, - "allow_query": "true" + "visibility": "hidden" }, "type": "string" }, diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event_hits_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event_hits_log.json new file mode 100644 index 0000000..5e3ff8a --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/proxy_event_hits_log.json @@ -0,0 +1,58 @@ +{ + "type": "record", + "name": "proxy_event_hits_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "action", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "hits", + "type": "long" + }, + { + "name": "policy_id", + "type": "long" + }, + { + "name": "sub_action", + "type": "string" + }, + { + "name": "country", + "type": "string" + }, + { + "name": "location", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "ip_object", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/public_schema_info.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/public_schema_info.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/public_schema_info.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/public_schema_info.json index 9839b49..7dd0feb 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/public_schema_info.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/public_schema_info.json @@ -1973,4 +1973,4 @@ ] } } -} +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log.json new file mode 100644 index 0000000..4f5e8d5 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "query_log", + "fields": [ + { + "name": "query_id", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log_cluster.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log_cluster.json new file mode 100644 index 0000000..d6e7583 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/query_log_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "query_log_cluster", + "fields": [ + { + "name": "type", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_onff_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_onff_log.json new file mode 100644 index 0000000..9201ebb --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_onff_log.json @@ -0,0 +1,37 @@ +{ + "type": "record", + "name": "radius_onff_log", + "namespace": "tsg_galaxy_v3", + "fields": [ + { + "name": "event_timestamp", + "label": "Event Time", + "type": "long" + }, + { + "name": "account", + "label": "Account", + "type": "string" + }, + { + "name": "framed_ip", + "label": "Framed IP", + "type": "string" + }, + { + "name": "acct_session_id", + "label": "Acct Session ID", + "type": "string" + }, + { + "name": "acct_status_type", + "label": "Acct Status Type", + "type": "int" + }, + { + "name": "acct_session_time", + "label": "Acct Session Time", + "type": "int" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/radius_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_record.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/radius_record.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_record.json index 58a7f37..eed1066 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/radius_record.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/radius_record.json @@ -615,7 +615,7 @@ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, - "allow_query": "true" + "visibility": "hidden" }, "type": "string" }, diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/recommendation_app_cip.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/recommendation_app_cip.json similarity index 100% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/recommendation_app_cip.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/recommendation_app_cip.json diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/relation_account_framedip.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/relation_account_framedip.json new file mode 100644 index 0000000..0ed7f7b --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/relation_account_framedip.json @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "relation_account_framedip", + "namespace": "tsg_galaxy", + "fields": [ + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/report_result.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/report_result.json new file mode 100644 index 0000000..7bc56e2 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/report_result.json @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "report_result", + "namespace": "tsg", + "fields": [ + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/security_event.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/security_event.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event.json index ce7579e..f3cf545 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/security_event.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event.json @@ -307,7 +307,9 @@ "internal_columns": [ "common_recv_time", "common_log_id", - "common_processing_time" + "common_processing_time", + "common_userdefine_app_name", + "common_tunnels" ], "tunnel_type": { "$ref": "public_schema_info.json#/tunnel_type" @@ -809,7 +811,7 @@ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, - "allow_query": "true" + "visibility": "hidden" }, "type": "string" }, diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event_hits_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event_hits_log.json new file mode 100644 index 0000000..c445244 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/security_event_hits_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "security_event_hits_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "action", + "type": "long" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "hits", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "policy_id", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/session_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/session_record.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record.json index 726761c..86a2946 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/session_record.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record.json @@ -282,7 +282,9 @@ "internal_columns": [ "common_recv_time", "common_log_id", - "common_processing_time" + "common_processing_time", + "common_userdefine_app_name", + "common_tunnels" ], "tunnel_type": { "$ref": "public_schema_info.json#/tunnel_type" @@ -778,7 +780,7 @@ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, - "allow_query": "true" + "visibility": "hidden" }, "type": "string" }, diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_client_ip.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_client_ip.json new file mode 100644 index 0000000..9184e36 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_client_ip.json @@ -0,0 +1,71 @@ +{ + "type": "record", + "name": "session_record_common_client_ip", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "common_client_ip" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "common_schema_type", + "type": "string" + }, + { + "name": "common_client_port", + "type": "int" + }, + { + "name": "common_server_port", + "type": "int" + }, + { + "name": "common_app_label", + "type": "string" + }, + { + "name": "common_direction", + "type": "int" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_server_ip.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_server_ip.json new file mode 100644 index 0000000..a7c977f --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_common_server_ip.json @@ -0,0 +1,71 @@ +{ + "type": "record", + "name": "session_record_common_server_ip", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "common_server_ip" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "common_schema_type", + "type": "string" + }, + { + "name": "common_client_port", + "type": "int" + }, + { + "name": "common_server_port", + "type": "int" + }, + { + "name": "common_app_label", + "type": "string" + }, + { + "name": "common_direction", + "type": "int" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_http_domain.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_http_domain.json new file mode 100644 index 0000000..65414ea --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/session_record_http_domain.json @@ -0,0 +1,71 @@ +{ + "type": "record", + "name": "session_record_http_domain", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "http_domain" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "common_schema_type", + "type": "string" + }, + { + "name": "common_client_port", + "type": "int" + }, + { + "name": "common_server_port", + "type": "int" + }, + { + "name": "common_app_label", + "type": "string" + }, + { + "name": "common_direction", + "type": "int" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_packet_capture_event.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_packet_capture_event.json new file mode 100644 index 0000000..47879de --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_packet_capture_event.json @@ -0,0 +1,801 @@ +{ + "type": "record", + "name": "sys_packet_capture_event", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time" + }, + "fields": [ + { + "name": "common_recv_time", + "type": "long", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "label": "Receive Time" + }, + { + "name": "common_log_id", + "type": "long", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "label": "Log ID" + }, + { + "name": "common_policy_id", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Policy ID" + }, + { + "name": "common_subscriber_id", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Subscriber ID" + }, + { + "name": "common_imei", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "IMEI" + }, + { + "name": "common_imsi", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "IMSI" + }, + { + "name": "common_phone_number", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Phone Number" + }, + { + "name": "common_client_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Client IP" + }, + { + "name": "common_internal_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Internal IP" + }, + { + "name": "common_client_port", + "type": "int", + "label": "Client Port" + }, + { + "name": "common_l4_protocol", + "type": "string", + "label": "L4 Protocol" + }, + { + "name": "common_address_type", + "type": "int", + "doc": { + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "label": "Address Type" + }, + { + "name": "common_server_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Server IP" + }, + { + "name": "common_server_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Server Port" + }, + { + "name": "common_external_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "External IP" + }, + { + "name": "common_action", + "type": "int", + "doc": { + "allow_query": "true", + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "label": "Action" + }, + { + "name": "common_direction", + "type": "int", + "doc": { + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "label": "Direction" + }, + { + "name": "common_entrance_id", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Entrance ID" + }, + { + "name": "common_sled_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Sled IP" + }, + { + "name": "common_client_location", + "type": "string", + "label": "Client Location" + }, + { + "name": "common_client_asn", + "type": "string", + "label": "Client ASN" + }, + { + "name": "common_server_location", + "type": "string", + "label": "Server Location" + }, + { + "name": "common_server_asn", + "type": "string", + "label": "Server ASN" + }, + { + "name": "common_sessions", + "type": "long", + "label": "Sessions" + }, + { + "name": "common_c2s_pkt_num", + "type": "long", + "label": "Packets Sent" + }, + { + "name": "common_s2c_pkt_num", + "type": "long", + "label": "Packets Received" + }, + { + "name": "common_c2s_byte_num", + "type": "long", + "label": "Bytes Sent" + }, + { + "name": "common_s2c_byte_num", + "type": "long", + "label": "Bytes Received" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "type": "long" + }, + { + "name": "common_service", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Service" + }, + { + "name": "common_schema_type", + "type": "string", + "doc": { + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "FTP", + "value": "FTP" + } + ], + "visibility": "hidden" + }, + "label": "Schema Type" + }, + { + "name": "common_user_tags", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "User Tags" + }, + { + "name": "common_sub_action", + "type": "string", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "label": "Sub Action" + }, + { + "name": "common_user_region", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "User Region" + }, + { + "name": "common_device_id", + "type": "string", + "label": "Device ID" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "ISP" + }, + { + "name": "common_device_tag", + "type": "string", + "doc": { + "visibility": "hidden", + "format": { + "functions": "flattenSpec,flattenSpec", + "appendTo": "common_data_center,common_device_group", + "param": "$.tags[?(@.tag=='data_center')].value,$.tags[?(@.tag=='device_group')].value" + } + }, + "label": "Device Tag" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + } + }, + "type": "string" + }, + { + "name": "common_device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + } + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ] + }, + "label": "Encapsulation" + }, + { + "name": "common_app_label", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Application Label" + }, + { + "name": "common_tunnels", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Tunnels" + }, + { + "name": "common_protocol_label", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Protocol Label" + }, + { + "name": "common_app_id", + "type": "string", + "label": "Application ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "type": "string", + "label": "Surrogate ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "type": "string", + "label": "L7 Protocol" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "label": "Start Time" + }, + { + "name": "common_end_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "label": "End Time" + }, + { + "name": "common_establish_latency_ms", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Establish Latency(ms)" + }, + { + "name": "common_con_duration_ms", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Duration(ms)" + }, + { + "name": "common_stream_dir", + "type": "int", + "doc": { + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "label": "Stream Direction" + }, + { + "name": "common_address_list", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Address List" + }, + { + "name": "common_has_dup_traffic", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "label": "Duplication Traffic" + }, + { + "name": "common_stream_error", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Stream Error" + }, + { + "name": "common_stream_trace_id", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "Session ID" + }, + { + "name": "common_link_info_c2s", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(c2s)" + }, + { + "name": "common_link_info_s2c", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(s2c)" + }, + { + "name": "common_c2s_ipfrag_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Fragmentation Packets(c2s)" + }, + { + "name": "common_s2c_ipfrag_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Fragmentation Packets(s2c)" + }, + { + "name": "common_c2s_tcp_lostlen", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Sequence Gap Loss(c2s)" + }, + { + "name": "common_s2c_tcp_lostlen", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Sequence Gap Loss(s2c)" + }, + { + "name": "common_c2s_tcp_unorder_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Unorder Packets(c2s)" + }, + { + "name": "common_s2c_tcp_unorder_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Unorder Packets(s2c)" + }, + { + "name": "common_c2s_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(c2s)" + }, + { + "name": "common_s2c_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(s2c)" + }, + { + "name": "common_c2s_byte_retrans", + "type": "long", + "label": "Byte Retransmission(c2s)" + }, + { + "name": "common_s2c_byte_retrans", + "type": "long", + "label": "Byte Retransmission(s2c)" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "First TTL" + }, + { + "name": "common_processing_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "label": "Processing Time" + }, + { + "name": "common_mirrored_pkts", + "label": "Mirrored Packets", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_mirrored_bytes", + "label": "Mirrored Bytes", + "type": "long", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "nic_name", + "type": "string", + "label": "Nic Name" + }, + { + "name": "origin_source_mac", + "type": "string", + "label": "Origin Source Mac" + }, + { + "name": "origin_dest_mac", + "type": "string", + "label": "Origin Dest Mac" + }, + { + "name": "packet_url", + "type": "string", + "label": "Packet URL" + }, + { + "name": "pcap_storage_task_id", + "type": "int", + "label": "Task ID" + }, + { + "name": "pcap_storage_duration", + "type": "int", + "label": "Duration" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_storage_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_storage_log.json new file mode 100644 index 0000000..3bb3224 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/sys_storage_log.json @@ -0,0 +1,38 @@ +{ + "type": "record", + "name": "sys_storage_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "log_type", + "type": "string" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "max_size", + "type": "long" + }, + { + "name": "used_size", + "type": "long" + }, + { + "name": "aggregate_size", + "type": "long" + }, + { + "name": "last_storage", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/tables_cluster.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/tables_cluster.json new file mode 100644 index 0000000..4765d85 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/tables_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "tables_cluster", + "fields": [ + { + "name": "database", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_client_ip_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_client_ip_log.json new file mode 100644 index 0000000..78f3867 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_client_ip_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_client_ip_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "source", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_external_host_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_external_host_log.json new file mode 100644 index 0000000..68c229e --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_external_host_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_external_host_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "destination", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_internal_host_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_internal_host_log.json new file mode 100644 index 0000000..75347a5 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_internal_host_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_internal_host_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "source", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_server_ip_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_server_ip_log.json new file mode 100644 index 0000000..74258f1 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_server_ip_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_server_ip_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "destination", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_urls_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_urls_log.json new file mode 100644 index 0000000..7a0cc9b --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_urls_log.json @@ -0,0 +1,22 @@ +{ + "type": "record", + "name": "top_urls_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "url", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_user_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_user_log.json new file mode 100644 index 0000000..ebddb24 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_user_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_user_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "subscriber_id", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_website_domain_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_website_domain_log.json new file mode 100644 index 0000000..df86ea9 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/top_website_domain_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_website_domain_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "domain", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_app_stat_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_app_stat_log.json new file mode 100644 index 0000000..083ef7b --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_app_stat_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "traffic_app_stat_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "app_name", + "type": "string" + }, + { + "name": "sub_app_name", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_metrics_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_metrics_log.json new file mode 100644 index 0000000..82b1ae1 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_metrics_log.json @@ -0,0 +1,218 @@ +{ + "type": "record", + "name": "traffic_metrics_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "allow_conn_num", + "type": "long" + }, + { + "name": "allow_in_bytes", + "type": "long" + }, + { + "name": "allow_in_packets", + "type": "long" + }, + { + "name": "allow_out_bytes", + "type": "long" + }, + { + "name": "allow_out_packets", + "type": "long" + }, + { + "name": "close_conn_num", + "type": "long" + }, + { + "name": "default_conn_num", + "type": "long" + }, + { + "name": "default_in_bytes", + "type": "long" + }, + { + "name": "default_in_packets", + "type": "long" + }, + { + "name": "default_out_bytes", + "type": "long" + }, + { + "name": "default_out_packets", + "type": "long" + }, + { + "name": "deny_conn_num", + "type": "long" + }, + { + "name": "deny_in_bytes", + "type": "long" + }, + { + "name": "deny_in_packets", + "type": "long" + }, + { + "name": "deny_out_bytes", + "type": "long" + }, + { + "name": "deny_out_packets", + "type": "long" + }, + { + "name": "device_id", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "intercept_conn_num", + "type": "long" + }, + { + "name": "intercept_in_bytes", + "type": "long" + }, + { + "name": "intercept_in_packets", + "type": "long" + }, + { + "name": "intercept_out_bytes", + "type": "long" + }, + { + "name": "intercept_out_packets", + "type": "long" + }, + { + "name": "established_conn_num", + "type": "long" + }, + { + "name": "monitor_conn_num", + "type": "long" + }, + { + "name": "monitor_in_bytes", + "type": "long" + }, + { + "name": "monitor_in_packets", + "type": "long" + }, + { + "name": "monitor_out_bytes", + "type": "long" + }, + { + "name": "monitor_out_packets", + "type": "long" + }, + { + "name": "new_conn_num", + "type": "long" + }, + { + "name": "total_in_bytes", + "type": "long" + }, + { + "name": "total_in_packets", + "type": "long" + }, + { + "name": "total_out_bytes", + "type": "long" + }, + { + "name": "total_out_packets", + "type": "long" + }, + { + "name": "alert_bytes", + "type": "long" + }, + { + "name": "hijk_bytes", + "type": "long" + }, + { + "name": "ins_bytes", + "type": "long" + }, + { + "name": "intcp_allow_num", + "type": "long" + }, + { + "name": "intcp_deny_num", + "type": "long" + }, + { + "name": "intcp_hijk_num", + "type": "long" + }, + { + "name": "intcp_ins_num", + "type": "long" + }, + { + "name": "intcp_mon_num", + "type": "long" + }, + { + "name": "intcp_rdirt_num", + "type": "long" + }, + { + "name": "intcp_repl_num", + "type": "long" + }, + { + "name": "maybe_pinning_num", + "type": "long" + }, + { + "name": "not_pinning_num", + "type": "long" + }, + { + "name": "pinning_num", + "type": "long" + }, + { + "name": "ad_cc_bytes", + "type": "long" + }, + { + "name": "ad_flood_bytes", + "type": "long" + }, + { + "name": "ad_reflection_bytes", + "type": "long" + }, + { + "name": "intcp_edit_elem_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_protocol_stat_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_protocol_stat_log.json new file mode 100644 index 0000000..d37b603 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_protocol_stat_log.json @@ -0,0 +1,78 @@ +{ + "type": "record", + "name": "traffic_protocol_stat_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "protocol_id", + "type": "string" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "device_group", + "type": "string" + }, + { + "name": "sessions", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "c2s_ipfrag_num", + "type": "long" + }, + { + "name": "s2c_ipfrag_num", + "type": "long" + }, + { + "name": "c2s_tcp_lostlen", + "type": "long" + }, + { + "name": "s2c_tcp_lostlen", + "type": "long" + }, + { + "name": "c2s_tcp_unorder_num", + "type": "long" + }, + { + "name": "s2c_tcp_unorder_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_summary_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_summary_log.json new file mode 100644 index 0000000..26088b8 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_summary_log.json @@ -0,0 +1,78 @@ +{ + "type": "record", + "name": "traffic_summary_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "device_group", + "type": "string" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "schema_type", + "type": "string" + }, + { + "name": "ip_object", + "type": "string" + }, + { + "name": "sessions", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "one_sided_connections", + "type": "long" + }, + { + "name": "uncategorized_bytes", + "type": "long" + }, + { + "name": "fragmentation_packets", + "type": "long" + }, + { + "name": "sequence_gap_loss", + "type": "long" + }, + { + "name": "unorder_packets", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_top_destination_ip_metrics_log.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_top_destination_ip_metrics_log.json new file mode 100644 index 0000000..ece6294 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/traffic_top_destination_ip_metrics_log.json @@ -0,0 +1,46 @@ +{ + "type": "record", + "name": "traffic_top_destination_ip_metrics_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_data_center", + "type": "string" + }, + { + "name": "destination_ip", + "type": "string" + }, + { + "name": "attack_type", + "type": "string" + }, + { + "name": "session_rate", + "type": "long" + }, + { + "name": "packet_rate", + "type": "long" + }, + { + "name": "bit_rate", + "type": "long" + }, + { + "name": "partition_num", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/transaction_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/transaction_record.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/transaction_record.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/transaction_record.json index 43e0a85..5d23390 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/transaction_record.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/transaction_record.json @@ -255,7 +255,8 @@ "internal_columns": [ "common_recv_time", "common_log_id", - "common_processing_time" + "common_processing_time", + "common_tunnels" ], "tunnel_type": { "$ref": "public_schema_info.json#/tunnel_type" @@ -719,7 +720,7 @@ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, - "allow_query": "true" + "visibility": "hidden" }, "type": "string" }, diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/version.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/version.json new file mode 100644 index 0000000..19fd977 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/version.json @@ -0,0 +1,95 @@ +{ + "product": "185 Environment", + "version": "21.11", + "registered": "Geedge", + "updated": "2021-08-01 00:00:00", + "components": { + "oss": [ + { + "name": "zookeeper", + "version": "3.4.10", + "licenseType": "Apache License 2.0", + "description": "分布式应用程序协调服务" + }, + { + "name": "kafka", + "version": "2.11_1.0.0", + "licenseType": "Apache License 2.0", + "description": "消息队列" + }, + { + "name": "habse", + "version": "2.2.3", + "licenseType": "Apache License 2.0", + "description": "用于文件系统和存储Radius数据" + }, + { + "name": "flume", + "version": "1.9.0", + "licenseType": "Apache License 2.0", + "description": "日志补全传输" + }, + { + "name": "clickhouse", + "version": "20.3.12.112-cluster", + "licenseType": "Apache License 2.0", + "description": "原始日志数据库" + }, + { + "name": "druid", + "version": "0.18.1", + "licenseType": "Apache License 2.0", + "description": "分析实时数据并提供低延迟查询的OLAP应用程序" + }, + { + "name": "gohangout", + "version": "1.15.2.20210408", + "description": "动态获取原始日志表schema入库程序" + } + ], + "apps": [ + { + "name": "galaxy-qgw-service", + "version": "345", + "description": "数据平台对外统一查询网关" + }, + { + "name": "galaxy-report-service", + "version": "21.04.07", + "description": "自定义报表查询服务" + }, + { + "name": "galaxy-hos-service", + "version": "21.07.01", + "description": "对象存储服务" + }, + { + "name": "xxl-job-admin", + "version": "v1.3.20210408", + "description": "分布式任务调度平台" + }, + { + "name": "xxl-job", + "version": "v1.3.210413-rc1", + "description": "分布式任务调度平台-执行器" + } + ], + "tasks": [ + { + "name": "flume", + "version": "flume-config-20.08", + "description": "原始日志补全、subscriber更新、Radius上下线功能" + }, + { + "name": "druid", + "version": "druid-config-20.08", + "description": "所有分析日志任务" + }, + { + "name": "gohangout", + "version": "gohangout-config-20.08", + "description": "原始日志入库、上下线日志入库" + } + ] + } +} \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/schema/voip_record.json b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/voip_record.json similarity index 99% rename from TSG发布版本更新记录/TSG-21.12/qgw/schema/voip_record.json rename to TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/voip_record.json index a3748c8..a27ee88 100644 --- a/TSG发布版本更新记录/TSG-21.12/qgw/schema/voip_record.json +++ b/TSG发布版本更新记录/TSG-21.12/qgw/galaxy-qgw-service/config/voip_record.json @@ -703,7 +703,7 @@ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, - "allow_query": "true" + "visibility": "hidden" }, "type": "string" }, diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/push_config.sh b/TSG发布版本更新记录/TSG-21.12/qgw/push_config.sh new file mode 100644 index 0000000..e9e5349 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/push_config.sh @@ -0,0 +1,48 @@ +#!/bin/bash +#修改config_path_root路径为放置要上传nacos的配置，IP地址为nacos地址 + +config_path_root=/home/deploy/bak/galaxy-qgw-service/config +tenant="prod" +nacos_push_url="http://192.168.44.67:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy" +retry=3 + +#1.获取config 路径 +config_path_list=$(readlink -f $config_path_root) + +function pushConfig(){ +#2.app文件夹文件 +for config_path in $config_path_list +do + if [ -d $config_path ];then + appPath=${config_path%/*} + appName=${appPath##*/} + for file in `ls $config_path` + do + config_path_file=$config_path/$file + if [ -f $config_path_file ];then + data_id=${config_path_file##*/} + suffix=${config_path_file##*.} + if [ "$suffix" = "sql" ];then + suffix="text" + fi + if [ "$suffix" = "yml" ];then + suffix="yaml" + fi + + for (( i = 0; i < 3; i++ )); do + if [ "$suffix" = "yaml" ]||[ "$suffix" = "json" ]||[ "$suffix" = "text" ]||[ "$suffix" = "properties" ];then + result=$(curl -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$appName&type=$suffix" --data-urlencode content="`cat $config_path_file`") + echo "push config response code "$result + if [[ $result -eq '200' ]];then + i=3 + fi + fi + done + + fi + done + fi +done +} + +pushConfig diff --git a/TSG发布版本更新记录/TSG-21.12/qgw/update-galaxy-qgw-service.yml b/TSG发布版本更新记录/TSG-21.12/qgw/update-galaxy-qgw-service.yml new file mode 100644 index 0000000..e9f721f --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/qgw/update-galaxy-qgw-service.yml @@ -0,0 +1,20 @@ +#在galaxy-qgw-service.yml 配置最下增加以下配置 + +## job cfg +job: + interactive: + timeout: 30000 + response: + timeout: 500 + execution: + timeout: 300000 + pool: + corePoolSize: 20 + maxPoolSize: 30 + queueCapacity: 10 +## task cfg +task: + pool: + corePoolSize: 8 + maxPoolSize: 16 + queueCapacity: 1000 \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/record-21.12.txt b/TSG发布版本更新记录/TSG-21.12/record-21.12.txt index 612cde0..0c14216 100644 --- a/TSG发布版本更新记录/TSG-21.12/record-21.12.txt +++ b/TSG发布版本更新记录/TSG-21.12/record-21.12.txt @@ -1,21 +1,24 @@ -21.12更新记录，按顺序更新 +21.12更新记录，按顺序更新，具体更新文件详见相关目录 kafka 1.新增topic TRAFFIC-APP-STAT +clickhouse +1.新增字段，clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password 进入ck客户端执行SQL文件里的SQL +2.修改clickhouse-server/config.xml中配置 + + hbase 1.新增hbase表 2.新增Phoenix视图 -clickhouse -1.新增字段，进入ck客户端执行SQL文件里的SQL - druid 1.更新Apache Druid任务traffic_app_stat_log, traffic_metrics_log +2.druid所有_common/目录下增加log4j2.component.properties 网关 1.更新镜像350 -2.更新schema 15个 +2.更新schema,使用backup_config.sh备份，push_config.sh提交配置到nacos 预处理 1.更新补全jar包及配置 @@ -25,12 +28,19 @@ druid 5.新增vpn-recommend任务 hos -1.更新镜像21.12.01 +1.更新镜像21.12.20 + +report +1.更新镜像21-12-15 flink +1.修改临时目录 +2.conf目录增加log4j2.component.properties spark -1.修改环境变量 +1.修改临时目录 +gohangout +1.更新config下所有任务配置max.partition.fetch.bytes: '31457280' 执行完成后重启所有gohangout \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/spark/update-spark-conf.txt b/TSG发布版本更新记录/TSG-21.12/spark/update-spark-conf.txt index ae48648..5b31342 100644 --- a/TSG发布版本更新记录/TSG-21.12/spark/update-spark-conf.txt +++ b/TSG发布版本更新记录/TSG-21.12/spark/update-spark-conf.txt @@ -1,2 +1,7 @@ + +#优化spark 使用脚本重启集群失败问题（pid存到/tmp目录） + spark-2.2.3-bin-hadoop2.7/conf/spark-env.sh -新增：export SPARK_PID_DIR=${SPARK_HOME}/tmp \ No newline at end of file +新增以下配置项： + +export SPARK_PID_DIR=${SPARK_HOME}/tmp \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/topology/README.txt b/TSG发布版本更新记录/TSG-21.12/topology/README.txt new file mode 100644 index 0000000..8dd9b16 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/topology/README.txt @@ -0,0 +1,9 @@ +flink任务更新步骤，顺序不分先后 + +注：修改各个更新的文本文件中ip地址等变量配置为线上实际值 + 启停任务使用各个目录下stop.sh ,start.sh + +1.新增account-framedip-Hbase任务，停调accountHbase，frameip任务 +2.更新dos-detection任务jar包，config/下配置新增update-config.txt中配置项 +3.更新flink-top任务jar包及kafka-flinksql-top.sql文件 +4.新增vpn-recommend任务 \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 b/TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 similarity index 100% rename from TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 rename to TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/config/RADIUS-RELATIONSHIP-HBASE-V2 diff --git a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/start.sh b/TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/start.sh similarity index 100% rename from TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/start.sh rename to TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/start.sh diff --git a/TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/stop.sh b/TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/stop.sh similarity index 100% rename from TSG发布版本更新记录/TSG-21.12/flink/account-framedip-Hbase/stop.sh rename to TSG发布版本更新记录/TSG-21.12/topology/account-framedip-Hbase/stop.sh diff --git a/TSG发布版本更新记录/TSG-21.12/topology/dos-detection/update-config b/TSG发布版本更新记录/TSG-21.12/topology/dos-detection/update-config new file mode 100644 index 0000000..91cdb1f --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.12/topology/dos-detection/update-config @@ -0,0 +1,4 @@ +# dos-detection任务config中配置新增以下配置 + +#baseline ttl，单位：天 +hbase.baseline.ttl=30 diff --git a/TSG发布版本更新记录/TSG-21.12/flink/topN/kafka-flinksql-top.sql b/TSG发布版本更新记录/TSG-21.12/topology/flink-top/kafka-flinksql-top.sql similarity index 88% rename from TSG发布版本更新记录/TSG-21.12/flink/topN/kafka-flinksql-top.sql rename to TSG发布版本更新记录/TSG-21.12/topology/flink-top/kafka-flinksql-top.sql index 877ac27..7f5c6dd 100644 --- a/TSG发布版本更新记录/TSG-21.12/flink/topN/kafka-flinksql-top.sql +++ b/TSG发布版本更新记录/TSG-21.12/topology/flink-top/kafka-flinksql-top.sql @@ -10,8 +10,8 @@ common_l4_protocol VARCHAR, common_internal_ip VARCHAR, common_external_ip VARCHAR, common_subscriber_id VARCHAR, -common_app_label VARCHAR, common_sessions BIGINT, +common_app_label VARCHAR, common_c2s_pkt_num BIGINT, common_s2c_pkt_num BIGINT, common_c2s_byte_num BIGINT, @@ -23,10 +23,10 @@ WITH( 'connector' = 'kafka', 'properties.group.id' = 'kafka-indexing-service', 'topic' = 'SESSION-RECORD-COMPLETED', -'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094', 'properties.security.protocol'='SASL_PLAINTEXT', 'properties.sasl.mechanism'='PLAIN', -'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', 'scan.startup.mode' = 'latest-offset', 'sink.parallelism'='1', --'sink.parallelism'='60', @@ -47,10 +47,10 @@ PRIMARY KEY (stat_time) NOT ENFORCED )WITH( 'connector' = 'upsert-kafka', 'topic' = 'TOP-CLIENT-IP', -'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094', 'properties.security.protocol'='SASL_PLAINTEXT', 'properties.sasl.mechanism'='PLAIN', -'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', --'sink.parallelism'='1', 'key.format' = 'json', 'value.format' = 'json' @@ -87,9 +87,6 @@ FROM top_client_ip_view) WHERE rownum <= 1000); - - - --server: CREATE TABLE top_server_ip_log( destination VARCHAR, @@ -104,10 +101,10 @@ PRIMARY KEY (stat_time) NOT ENFORCED )WITH( 'connector' = 'upsert-kafka', 'topic' = 'TOP-SERVER-IP', -'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094', 'properties.security.protocol'='SASL_PLAINTEXT', 'properties.sasl.mechanism'='PLAIN', -'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', --'sink.parallelism'='1', 'key.format' = 'json', 'value.format' = 'json' @@ -144,7 +141,6 @@ FROM top_server_ip_view) WHERE rownum <= 1000); - --internal CREATE TABLE top_internal_ip_log ( source VARCHAR, @@ -159,10 +155,10 @@ CREATE TABLE top_internal_ip_log ( ) WITH ( 'connector' = 'upsert-kafka', 'topic' = 'TOP-INTERNAL-HOST', -'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094', 'properties.security.protocol'='SASL_PLAINTEXT', 'properties.sasl.mechanism'='PLAIN', -'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', --'sink.parallelism'='1', 'key.format' = 'json', 'value.format' = 'json' @@ -215,10 +211,10 @@ CREATE TABLE top_external_ip_log ( ) WITH ( 'connector' = 'upsert-kafka', 'topic' = 'TOP-EXTERNAL-HOST', -'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094', 'properties.security.protocol'='SASL_PLAINTEXT', 'properties.sasl.mechanism'='PLAIN', -'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', --'sink.parallelism'='1', 'key.format' = 'json', 'value.format' = 'json' @@ -257,7 +253,6 @@ FROM top_common_external_ip_view) WHERE rownum <= 1000); - --website_domain CREATE TABLE top_website_domain_log ( domain VARCHAR, @@ -272,10 +267,10 @@ CREATE TABLE top_website_domain_log ( ) WITH ( 'connector' = 'upsert-kafka', 'topic' = 'TOP-WEBSITE-DOMAIN', -'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094', 'properties.security.protocol'='SASL_PLAINTEXT', 'properties.sasl.mechanism'='PLAIN', -'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', --'sink.parallelism'='1', 'key.format' = 'json', 'value.format' = 'json' @@ -328,10 +323,10 @@ CREATE TABLE top_user_log ( ) WITH ( 'connector' = 'upsert-kafka', 'topic' = 'TOP-USER', -'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.13:9094,192.168.44.15:9094,192.168.44.16:9094', 'properties.security.protocol'='SASL_PLAINTEXT', 'properties.sasl.mechanism'='PLAIN', -'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', --'sink.parallelism'='1', 'key.format' = 'json', 'value.format' = 'json' @@ -368,10 +363,6 @@ FROM top_user_log_view) WHERE rownum <= 1000); - - - - --app CREATE TABLE top_app_log ( app_name VARCHAR, @@ -385,11 +376,10 @@ CREATE TABLE top_app_log ( ) WITH ( 'connector' = 'upsert-kafka', 'topic' = 'TRAFFIC-APP-STAT', -'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094', ---'properties.bootstrap.servers' = '10.111.136.193:9092,10.111.136.194:9092,10.111.136.195:9092,10.111.136.196:9092,10.111.136.197:9092,10.111.136.198:9092,10.111.136.199:9092,10.111.136.200:9092,10.111.136.201:9092,10.111.136.203:9092,10.111.136.204:9092,10.111.136.205:9092,10.111.136.206:9092,10.111.136.207:9092,10.111.136.202:9092', +'properties.bootstrap.servers' = '192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094,192.168.44.16:9094', 'properties.security.protocol'='SASL_PLAINTEXT', 'properties.sasl.mechanism'='PLAIN', -'properties.sasl.jaas.config'= 'org.apache.flink.kafka.shaded.org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', +'properties.sasl.jaas.config'= 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="galaxy2019";', 'key.format' = 'json', 'value.format' = 'json' ); @@ -407,9 +397,3 @@ INSERT INTO top_app_log ROW_NUMBER() OVER (PARTITION BY stat_time ) FROM top_app_log_view)); - - - - - - diff --git a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/config/RECOMMENDATION-APP-CIP b/TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/config/RECOMMENDATION-APP-CIP similarity index 95% rename from TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/config/RECOMMENDATION-APP-CIP rename to TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/config/RECOMMENDATION-APP-CIP index 9e96c4e..d3b41a2 100644 --- a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/config/RECOMMENDATION-APP-CIP +++ b/TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/config/RECOMMENDATION-APP-CIP @@ -31,7 +31,7 @@ kafka.user=admin #kafka SASL及SSL验证密码 kafka.pin=galaxy2019 #1SSL需要 -tools.library=D:\\K18-Phase2\\tsgSpace\\dat\\tsg\\ +tools.library=/home/bigdata/topology/dat/ #是否接受全量app 无过滤条件false 白名单过滤true has.filter=false #只计算filter命中的common_app_label逗号分隔 baidu.com,qq 可不填写 diff --git a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/start.sh b/TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/start.sh similarity index 95% rename from TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/start.sh rename to TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/start.sh index 14a8aa4..082d8a2 100644 --- a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/start.sh +++ b/TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/start.sh @@ -1,5 +1,4 @@ -#! /bin/bash -#启动storm任务脚本 +#!/bin/bash source /etc/profile #任务jar所在目录 BASE_DIR=`pwd` diff --git a/TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/stop.sh b/TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/stop.sh similarity index 100% rename from TSG发布版本更新记录/TSG-21.12/flink/vpn-recommend/stop.sh rename to TSG发布版本更新记录/TSG-21.12/topology/vpn-recommend/stop.sh