From a9366e264e93d5899fe69b4a8948d1d6acf08b8a Mon Sep 17 00:00:00 2001 From: qidaijie Date: Thu, 9 Sep 2021 17:13:01 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4clickhouse=E5=92=8Cdruid?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../TSG-21.09/Clickhouse/bloom_filter.sql | 3 + .../TSG-21.09/Clickhouse/rename.sql | 2776 +++++++++++++++++ TSG发布版本更新记录/TSG-21.09/Druid/tasks.txt | 17 + .../TSG-21.09/Druid/tasks/proxy_event_hits_log.json | 96 + .../Druid/tasks/security_event_hits_log.json | 76 + .../TSG-21.09/Druid/tasks/sys_storage_log.json | 51 + .../TSG-21.09/Druid/tasks/top_client_ip_log.json | 52 + .../TSG-21.09/Druid/tasks/top_external_host_log.json | 53 + .../TSG-21.09/Druid/tasks/top_internal_host_log.json | 52 + .../TSG-21.09/Druid/tasks/top_server_ip_log.json | 53 + .../TSG-21.09/Druid/tasks/top_urls_log.json | 49 + .../TSG-21.09/Druid/tasks/top_user_log.json | 52 + .../TSG-21.09/Druid/tasks/top_website_domain_log.json | 53 + .../TSG-21.09/Druid/tasks/traffic_app_stat_log.json | 63 + .../TSG-21.09/Druid/tasks/traffic_metrics_log.json | 152 + .../Druid/tasks/traffic_protocol_stat_log.json | 65 + .../TSG-21.09/Druid/tasks/traffic_summary_log.json | 74 + .../tasks/traffic_top_destination_ip_metrics_log.json | 58 + .../TSG-21.09/Druid/tasks/urls_proxy_hot.json | 61 + .../TSG-21.09/Druid/tasks/urls_security_hot.json | 61 + .../TSG-21.09/{Clickhouse => Flink}/.gitkeep | 0 .../TSG-21.09/{Druid => Gohangout}/.gitkeep | 0 22 files changed, 3917 insertions(+) create mode 100644 TSG发布版本更新记录/TSG-21.09/Clickhouse/bloom_filter.sql create mode 100644 TSG发布版本更新记录/TSG-21.09/Clickhouse/rename.sql create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks.txt create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/proxy_event_hits_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/security_event_hits_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/sys_storage_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_client_ip_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_external_host_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_internal_host_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_server_ip_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_urls_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_user_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_website_domain_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_app_stat_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_metrics_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_protocol_stat_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_summary_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_top_destination_ip_metrics_log.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_proxy_hot.json create mode 100644 TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_security_hot.json rename TSG发布版本更新记录/TSG-21.09/{Clickhouse => Flink}/.gitkeep (100%) rename TSG发布版本更新记录/TSG-21.09/{Druid => Gohangout}/.gitkeep (100%) diff --git a/TSG发布版本更新记录/TSG-21.09/Clickhouse/bloom_filter.sql b/TSG发布版本更新记录/TSG-21.09/Clickhouse/bloom_filter.sql new file mode 100644 index 0000000..870a76e --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Clickhouse/bloom_filter.sql @@ -0,0 +1,3 @@ +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index common_client_ip type bloom_filter(0.05) GRANULARITY 1; \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.09/Clickhouse/rename.sql b/TSG发布版本更新记录/TSG-21.09/Clickhouse/rename.sql new file mode 100644 index 0000000..4069147 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Clickhouse/rename.sql @@ -0,0 +1,2776 @@ +DROP TABLE IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster ; +DROP TABLE IF EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster ; +DROP TABLE IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster ; + + +DROP TABLE IF EXISTS tsg_galaxy_v3.connection_record_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.live_session_record_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.transaction_record_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.radius_record_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.voip_record_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.gtpc_record_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.security_event_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.proxy_event_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.dos_event_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.active_defence_event_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.sys_packet_capture_log ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.connection_record_log_common_client_ip ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.connection_record_log_common_server_ip ON CLUSTER ck_all; +DROP TABLE IF EXISTS tsg_galaxy_v3.connection_record_log_http_domain ON CLUSTER ck_all; + + +RENAME table tsg_galaxy_v3.connection_record_log_local to tsg_galaxy_v3.session_record_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.live_session_record_log_local to tsg_galaxy_v3.interim_session_record_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.transaction_record_log_local to tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.radius_record_log_local to tsg_galaxy_v3.radius_record_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.voip_record_log_local to tsg_galaxy_v3.voip_record_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.gtpc_record_log_local to tsg_galaxy_v3.gtpc_record_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.security_event_log_local to tsg_galaxy_v3.security_event_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.proxy_event_log_local to tsg_galaxy_v3.proxy_event_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.dos_event_log_local to tsg_galaxy_v3.dos_event_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.active_defence_event_log_local to tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.sys_packet_capture_log_local to tsg_galaxy_v3.sys_packet_capture_event_local ON CLUSTER ck_cluster; + + + +RENAME table tsg_galaxy_v3.connection_record_log_common_client_ip_local to tsg_galaxy_v3.session_record_common_client_ip_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.connection_record_log_common_server_ip_local to tsg_galaxy_v3.session_record_common_server_ip_local ON CLUSTER ck_cluster; +RENAME table tsg_galaxy_v3.connection_record_log_http_domain_local to tsg_galaxy_v3.session_record_http_domain_local ON CLUSTER ck_cluster; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni,common_schema_type,common_client_port,common_server_port,common_app_label,common_direction FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni,common_schema_type,common_client_port,common_server_port,common_app_label,common_direction FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni,common_schema_type,common_client_port,common_server_port,common_app_label,common_direction FROM tsg_galaxy_v3.session_record_local; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event ON CLUSTER ck_cluster( + log_id UInt64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) ENGINE = Distributed('ck_cluster', 'tsg_galaxy_v3', 'dos_event_local', rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event ON CLUSTER ck_query( + log_id UInt64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) ENGINE = Distributed('ck_cluster', 'tsg_galaxy_v3', 'dos_event_local', rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,gtpc_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,gtpc_record_local,rand()); + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,interim_session_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,interim_session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_event_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location LowCardinality(String), + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location LowCardinality(String), + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label LowCardinality(String), + common_app_surrogate_id String, + common_l7_protocol LowCardinality(String), + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol LowCardinality(String), + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type LowCardinality(String), + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_mirrored_pkts Int64, + common_mirrored_bytes Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_event_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_http_domain_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_http_domain_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_local,rand()); + + + + + + + + + + + diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks.txt b/TSG发布版本更新记录/TSG-21.09/Druid/tasks.txt new file mode 100644 index 0000000..b2385d4 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks.txt @@ -0,0 +1,17 @@ +proxy_event_hits_log.json +security_event_hits_log.json +sys_storage_log.json +top_client_ip_log.json +top_external_host_log.json +top_internal_host_log.json +top_server_ip_log.json +top_urls_log.json +top_user_log.json +top_website_domain_log.json +traffic_metrics_log.json +traffic_protocol_stat_log.json +traffic_summary_log.json +traffic_top_destination_ip_metrics_log.json +urls_proxy_hot.json +urls_security_hot.json +traffic_app_stat_log.json diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/proxy_event_hits_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/proxy_event_hits_log.json new file mode 100644 index 0000000..b1350cd --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/proxy_event_hits_log.json @@ -0,0 +1,96 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "proxy_event_hits_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + { "name": "policy_id", "type": "long" }, + "isp", + "country", + "location", + { "name": "entrance_id", "type": "long" }, + { "name": "action", "type": "long" }, + "sub_action" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" }, + { "type" : "count", "name" : "hits" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "country", + "expression": "array_ordinal(string_to_array(common_client_location,','),3)" + }, + { + "type": "expression", + "name": "location", + "expression": "array_ordinal(string_to_array(common_client_location,','),2)" + }, + { + "type": "expression", + "name": "policy_id", + "expression": "common_policy_id" + }, + { + "type": "expression", + "name": "isp", + "expression": "common_isp" + }, + { + "type": "expression", + "name": "entrance_id", + "expression": "common_entrance_id" + }, + { + "type": "expression", + "name": "sub_action", + "expression": "common_sub_action" + }, + { + "type": "expression", + "name": "action", + "expression": "common_action" + } + ] + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 20000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "PROXY-EVENT-COMPLETED", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/security_event_hits_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/security_event_hits_log.json new file mode 100644 index 0000000..aef3396 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/security_event_hits_log.json @@ -0,0 +1,76 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "security_event_hits_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + { "name": "policy_id", "type": "long" }, + "isp", + { "name": "entrance_id", "type": "long" }, + { "name": "action", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "count", "name" : "hits" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "policy_id", + "expression": "common_policy_id" + }, + { + "type": "expression", + "name": "isp", + "expression": "common_isp" + }, + { + "type": "expression", + "name": "entrance_id", + "expression": "common_entrance_id" + }, + { + "type": "expression", + "name": "action", + "expression": "common_action" + } + ] + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SECURITY-EVENT-COMPLETED", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/sys_storage_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/sys_storage_log.json new file mode 100644 index 0000000..dbfdb9c --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/sys_storage_log.json @@ -0,0 +1,51 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sys_storage_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "log_type", + "data_center", + { "name": "used_size", "type": "long" }, + { "name": "max_size", "type": "long" }, + { "name": "aggregate_size", "type": "long" }, + { "name": "last_storage", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SYS-STORAGE", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_client_ip_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_client_ip_log.json new file mode 100644 index 0000000..6a15d30 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_client_ip_log.json @@ -0,0 +1,52 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_client_ip_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "source", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-CLIENT-IP", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_external_host_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_external_host_log.json new file mode 100644 index 0000000..cb023dd --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_external_host_log.json @@ -0,0 +1,53 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_external_host_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "destination", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-EXTERNAL-HOST", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_internal_host_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_internal_host_log.json new file mode 100644 index 0000000..85ecab7 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_internal_host_log.json @@ -0,0 +1,52 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_internal_host_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "source", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-INTERNAL-HOST", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_server_ip_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_server_ip_log.json new file mode 100644 index 0000000..884e874 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_server_ip_log.json @@ -0,0 +1,53 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_server_ip_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "destination", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-SERVER-IP", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_urls_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_urls_log.json new file mode 100644 index 0000000..6edeeec --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_urls_log.json @@ -0,0 +1,49 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_urls_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "url" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{} + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-URLS", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_user_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_user_log.json new file mode 100644 index 0000000..a1cfa70 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_user_log.json @@ -0,0 +1,52 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_user_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "subscriber_id", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-USER", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_website_domain_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_website_domain_log.json new file mode 100644 index 0000000..bb974d8 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/top_website_domain_log.json @@ -0,0 +1,53 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_website_domain_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "domain", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-WEBSITE-DOMAIN", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_app_stat_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_app_stat_log.json new file mode 100644 index 0000000..5267611 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_app_stat_log.json @@ -0,0 +1,63 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_app_stat_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "app_name", + "sub_app_name" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num","fieldName" : "common_sessions"}, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "app_name", "expression": "array_ordinal(string_to_array(replace(common_app_label, '.', '/'),'/'),1)"}, + {"type": "expression", "name": "sub_app_name", "expression": "array_ordinal(string_to_array(replace(common_app_label, '.', '/'),'/'),2)"} + ], + "filter": { + "type": "not", + "field":{ "type": "selector", "dimension": "app_name", "value": "" } + } + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SESSION-RECORD-COMPLETED", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "earlyMessageRejectionPeriod": "PT6H", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_metrics_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_metrics_log.json new file mode 100644 index 0000000..b3b0432 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_metrics_log.json @@ -0,0 +1,152 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_metrics_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "entrance_id" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "allow_conn_num", "type": "path", "expr": "$.fields.allow_conn_num" }, + { "name": "allow_in_bytes", "type": "path", "expr": "$.fields.allow_in_bytes" }, + { "name": "allow_in_packets", "type": "path", "expr": "$.fields.allow_in_packets" }, + { "name": "allow_out_bytes", "type": "path", "expr": "$.fields.allow_out_bytes" }, + { "name": "allow_out_packets", "type": "path", "expr": "$.fields.allow_out_packets" }, + { "name": "close_conn_num", "type": "path", "expr": "$.fields.close_conn_num" }, + { "name": "default_conn_num", "type": "path", "expr": "$.fields.default_conn_num" }, + { "name": "default_in_bytes", "type": "path", "expr": "$.fields.default_in_bytes" }, + { "name": "default_in_packets", "type": "path", "expr": "$.fields.default_in_packets" }, + { "name": "default_out_bytes", "type": "path", "expr": "$.fields.default_out_bytes" }, + { "name": "default_out_packets", "type": "path", "expr": "$.fields.default_out_packets" }, + { "name": "deny_conn_num", "type": "path", "expr": "$.fields.deny_conn_num" }, + { "name": "deny_in_bytes", "type": "path", "expr": "$.fields.deny_in_bytes" }, + { "name": "deny_in_packets", "type": "path", "expr": "$.fields.deny_in_packets" }, + { "name": "deny_out_bytes", "type": "path", "expr": "$.fields.deny_out_bytes" }, + { "name": "deny_out_packets", "type": "path", "expr": "$.fields.deny_out_packets" }, + { "name": "established_conn_num", "type": "path", "expr": "$.fields.established_conn_num" }, + { "name": "intercept_conn_num", "type": "path", "expr": "$.fields.intercept_conn_num" }, + { "name": "intercept_in_bytes", "type": "path", "expr": "$.fields.intercept_in_bytes" }, + { "name": "intercept_in_packets", "type": "path", "expr": "$.fields.intercept_in_packets" }, + { "name": "intercept_out_bytes", "type": "path", "expr": "$.fields.intercept_out_bytes" }, + { "name": "intercept_out_packets", "type": "path", "expr": "$.fields.intercept_out_packets" }, + { "name": "monitor_conn_num", "type": "path", "expr": "$.fields.monitor_conn_num" }, + { "name": "monitor_in_bytes", "type": "path", "expr": "$.fields.monitor_in_bytes" }, + { "name": "monitor_in_packets", "type": "path", "expr": "$.fields.monitor_in_packets" }, + { "name": "monitor_out_bytes", "type": "path", "expr": "$.fields.monitor_out_bytes" }, + { "name": "monitor_out_packets", "type": "path", "expr": "$.fields.monitor_out_packets" }, + { "name": "new_conn_num", "type": "path", "expr": "$.fields.new_conn_num" }, + { "name": "total_in_bytes", "type": "path", "expr": "$.fields.total_in_bytes" }, + { "name": "total_in_packets", "type": "path", "expr": "$.fields.total_in_packets" }, + { "name": "total_out_bytes", "type": "path", "expr": "$.fields.total_out_bytes" }, + { "name": "total_out_packets", "type": "path", "expr": "$.fields.total_out_packets" }, + { "name": "pinning_num", "type": "path", "expr": "$.fields.pinning_num" }, + { "name": "not_pinning_num", "type": "path", "expr": "$.fields.not_pinning_num" }, + { "name": "maybe_pinning_num", "type": "path", "expr": "$.fields.maybe_pinning_num" }, + { "name": "alert_bytes", "type": "path", "expr": "$.fields.alert_bytes" }, + { "name": "insert_bytes", "type": "path", "expr": "$.fields.insert_bytes" }, + { "name": "hijack_bytes", "type": "path", "expr": "$.fields.hijack_bytes" }, + { "name": "ad_reflection_bytes", "type": "path", "expr": "$.fields.ad_reflection_bytes" }, + { "name": "ad_flood_bytes", "type": "path", "expr": "$.fields.ad_flood_bytes" }, + { "name": "ad_cc_bytes", "type": "path", "expr": "$.fields.ad_cc_bytes" }, + { "name": "intercept_monitor_conn_num", "type": "path", "expr": "$.fields.intcp_mon_num" }, + { "name": "intercept_allow_conn_num", "type": "path", "expr": "$.fields.intcp_allow_num" }, + { "name": "intercept_deny_conn_num", "type": "path", "expr": "$.fields.intcp_deny_num" }, + { "name": "intercept_redirect_conn_num", "type": "path", "expr": "$.fields.intcp_rdirt_num" }, + { "name": "intercept_replace_conn_num", "type": "path", "expr": "$.fields.intcp_repl_num" }, + { "name": "intercept_hijack_conn_num", "type": "path", "expr": "$.fields.intcp_hijk_num" }, + { "name": "intercept_insert_conn_num", "type": "path", "expr": "$.fields.intcp_ins_num" }, + { "name": "app_name", "type": "path", "expr": "$.tags.app_name" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "entrance_id", "type": "path", "expr": "$.tags.entrance_id" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "new_conn_num", "fieldName" : "new_conn_num" }, + { "type" : "longMax", "name" : "established_conn_num", "fieldName" : "established_conn_num" }, + { "type" : "longSum", "name" : "close_conn_num", "fieldName" : "close_conn_num" }, + { "type" : "longSum", "name" : "total_in_bytes", "fieldName" : "total_in_bytes" }, + { "type" : "longSum", "name" : "total_out_bytes", "fieldName" : "total_out_bytes" }, + { "type" : "longSum", "name" : "total_in_packets", "fieldName" : "total_in_packets" }, + { "type" : "longSum", "name" : "total_out_packets", "fieldName" : "total_out_packets" }, + { "type" : "longSum", "name" : "default_conn_num", "fieldName" : "default_conn_num" }, + { "type" : "longSum", "name" : "default_in_bytes", "fieldName" : "default_in_bytes" }, + { "type" : "longSum", "name" : "default_out_bytes", "fieldName" : "default_out_bytes" }, + { "type" : "longSum", "name" : "default_in_packets", "fieldName" : "default_in_packets" }, + { "type" : "longSum", "name" : "default_out_packets", "fieldName" : "default_out_packets" }, + { "type" : "longSum", "name" : "allow_conn_num", "fieldName" : "allow_conn_num" }, + { "type" : "longSum", "name" : "allow_in_bytes", "fieldName" : "allow_in_bytes" }, + { "type" : "longSum", "name" : "allow_out_bytes", "fieldName" : "allow_out_bytes" }, + { "type" : "longSum", "name" : "allow_in_packets", "fieldName" : "allow_in_packets" }, + { "type" : "longSum", "name" : "allow_out_packets", "fieldName" : "allow_out_packets" }, + { "type" : "longSum", "name" : "deny_conn_num", "fieldName" : "deny_conn_num" }, + { "type" : "longSum", "name" : "deny_in_bytes", "fieldName" : "deny_in_bytes" }, + { "type" : "longSum", "name" : "deny_out_bytes", "fieldName" : "deny_out_bytes" }, + { "type" : "longSum", "name" : "deny_in_packets", "fieldName" : "deny_in_packets" }, + { "type" : "longSum", "name" : "deny_out_packets", "fieldName" : "deny_out_packets" }, + { "type" : "longSum", "name" : "monitor_conn_num", "fieldName" : "monitor_conn_num" }, + { "type" : "longSum", "name" : "monitor_in_bytes", "fieldName" : "monitor_in_bytes" }, + { "type" : "longSum", "name" : "monitor_out_bytes", "fieldName" : "monitor_out_bytes" }, + { "type" : "longSum", "name" : "monitor_in_packets", "fieldName" : "monitor_in_packets" }, + { "type" : "longSum", "name" : "monitor_out_packets", "fieldName" : "monitor_out_packets" }, + { "type" : "longSum", "name" : "intercept_conn_num", "fieldName" : "intercept_conn_num" }, + { "type" : "longSum", "name" : "intercept_in_bytes", "fieldName" : "intercept_in_bytes" }, + { "type" : "longSum", "name" : "intercept_out_bytes", "fieldName" : "intercept_out_bytes" }, + { "type" : "longSum", "name" : "intercept_in_packets", "fieldName" : "intercept_in_packets" }, + { "type" : "longSum", "name" : "intercept_out_packets", "fieldName" : "intercept_out_packets" }, + { "type" : "longSum", "name" : "pinning_num", "fieldName" : "pinning_num" }, + { "type" : "longSum", "name" : "not_pinning_num", "fieldName" : "not_pinning_num" }, + { "type" : "longSum", "name" : "maybe_pinning_num", "fieldName" : "maybe_pinning_num" }, + { "type" : "longSum", "name" : "alert_bytes", "fieldName" : "alert_bytes" }, + { "type" : "longSum", "name" : "ins_bytes", "fieldName" : "insert_bytes" }, + { "type" : "longSum", "name" : "hijk_bytes", "fieldName" : "hijack_bytes" }, + { "type" : "longSum", "name" : "ad_reflection_bytes", "fieldName" : "ad_reflection_bytes" }, + { "type" : "longSum", "name" : "ad_flood_bytes", "fieldName" : "ad_flood_bytes" }, + { "type" : "longSum", "name" : "ad_cc_bytes", "fieldName" : "ad_cc_bytes" }, + { "type" : "longSum", "name" : "intcp_mon_num", "fieldName" : "intercept_monitor_conn_num" }, + { "type" : "longSum", "name" : "intcp_allow_num", "fieldName" : "intercept_allow_conn_num" }, + { "type" : "longSum", "name" : "intcp_deny_num", "fieldName" : "intercept_deny_conn_num" }, + { "type" : "longSum", "name" : "intcp_rdirt_num", "fieldName" : "intercept_redirect_conn_num" }, + { "type" : "longSum", "name" : "intcp_repl_num", "fieldName" : "intercept_replace_conn_num" }, + { "type" : "longSum", "name" : "intcp_hijk_num", "fieldName" : "intercept_hijack_conn_num" }, + { "type" : "longSum", "name" : "intcp_ins_num", "fieldName" : "intercept_insert_conn_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5S", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{} + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-METRICS", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_protocol_stat_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_protocol_stat_log.json new file mode 100644 index 0000000..072d11d --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_protocol_stat_log.json @@ -0,0 +1,65 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_protocol_stat_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "protocol_id", + "isp", + "data_center", + { + "name": "entrance_id", + "type": "long" + } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" }, + { "type" : "longSum", "name" : "c2s_ipfrag_num", "fieldName" : "c2s_ipfrag_num" }, + { "type" : "longSum", "name" : "s2c_ipfrag_num", "fieldName" : "s2c_ipfrag_num" }, + { "type" : "longSum", "name" : "c2s_tcp_lostlen", "fieldName" : "c2s_tcp_lostlen" }, + { "type" : "longSum", "name" : "s2c_tcp_lostlen", "fieldName" : "s2c_tcp_lostlen" }, + { "type" : "longSum", "name" : "c2s_tcp_unorder_num", "fieldName" : "c2s_tcp_unorder_num" }, + { "type" : "longSum", "name" : "s2c_tcp_unorder_num", "fieldName" : "s2c_tcp_unorder_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{} + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-PROTOCOL-STAT", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_summary_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_summary_log.json new file mode 100644 index 0000000..ddb9700 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_summary_log.json @@ -0,0 +1,74 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_summary_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "isp", + "data_center", + "schema_type", + { "name": "entrance_id", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" }, + { "type" : "longSum", "name" : "sessions", "fieldName" : "common_sessions" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" }, + { "type" : "longSum", "name" : "one_sided_connections", "fieldName" : "one_sided_connections" }, + { "type" : "longSum", "name" : "uncategorized_bytes", "fieldName" : "uncategorized_bytes" }, + { "type" : "longSum", "name" : "fragmentation_packets", "fieldName" : "fragmentation_packets" }, + { "type" : "longSum", "name" : "sequence_gap_loss", "fieldName" : "sequence_gap_loss" }, + { "type" : "longSum", "name" : "unorder_packets", "fieldName" : "unorder_packets" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression","name": "isp","expression": "common_isp"}, + {"type": "expression","name": "data_center","expression": "common_data_center"}, + {"type": "expression","name": "entrance_id","expression": "common_entrance_id"}, + {"type": "expression","name": "schema_type","expression": "common_schema_type"}, + {"type": "expression", "name": "one_sided_connections", "expression": "if(common_stream_dir!=3,common_sessions,0)"}, + {"type": "expression", "name": "uncategorized_bytes", "expression": "if(array_ordinal(string_to_array(common_protocol_label,'/'),1)=='UNCATEGORIZED',\"common_c2s_byte_num\"+\"common_s2c_byte_num\",0)"}, + {"type": "expression","name": "fragmentation_packets","expression": "\"common_c2s_ipfrag_num\"+\"common_s2c_ipfrag_num\""}, + {"type": "expression","name": "sequence_gap_loss","expression": "\"common_c2s_tcp_lostlen\"+\"common_s2c_tcp_lostlen\""}, + {"type": "expression","name": "unorder_packets","expression": "\"common_c2s_tcp_unorder_num\"+\"common_s2c_tcp_unorder_num\""} + ] + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 20000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SESSION-RECORD-COMPLETED", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_top_destination_ip_metrics_log.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_top_destination_ip_metrics_log.json new file mode 100644 index 0000000..f1eb436 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/traffic_top_destination_ip_metrics_log.json @@ -0,0 +1,58 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_top_destination_ip_metrics_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "sketch_start_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "common_sled_ip", + "destination_ip", + "common_data_center", + "attack_type", + { "type": "long", "name": "partition_num" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"}, + { "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" }, + { "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" }, + { "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT10M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[] + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-DESTINATION-IP-METRICS", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_proxy_hot.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_proxy_hot.json new file mode 100644 index 0000000..7554d06 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_proxy_hot.json @@ -0,0 +1,61 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "proxy_event_urls_hot_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "url" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "count", "name" : "session_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "url", + "expression": "http_url" + } + ], + "filter": { + "type": "not", + "field":{ "type": "selector", "dimension": "url", "value": "" } + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "PROXY-EVENT-COMPLETED", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_security_hot.json b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_security_hot.json new file mode 100644 index 0000000..d92104d --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.09/Druid/tasks/urls_security_hot.json @@ -0,0 +1,61 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "security_event_urls_hot_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "url" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "count", "name" : "session_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "url", + "expression": "http_url" + } + ], + "filter": { + "type": "not", + "field":{ "type": "selector", "dimension": "url", "value": "" } + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SECURITY-EVENT-COMPLETED", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/TSG发布版本更新记录/TSG-21.09/Clickhouse/.gitkeep b/TSG发布版本更新记录/TSG-21.09/Flink/.gitkeep similarity index 100% rename from TSG发布版本更新记录/TSG-21.09/Clickhouse/.gitkeep rename to TSG发布版本更新记录/TSG-21.09/Flink/.gitkeep diff --git a/TSG发布版本更新记录/TSG-21.09/Druid/.gitkeep b/TSG发布版本更新记录/TSG-21.09/Gohangout/.gitkeep similarity index 100% rename from TSG发布版本更新记录/TSG-21.09/Druid/.gitkeep rename to TSG发布版本更新记录/TSG-21.09/Gohangout/.gitkeep