From 42af770e19af46e5a023d7ffbbb0f6fd407e12df Mon Sep 17 00:00:00 2001 From: lifengchao Date: Fri, 22 Mar 2024 11:33:24 +0800 Subject: [PATCH] =?UTF-8?q?TSG=202307=E7=89=88=E6=9C=AC=E5=88=B02402?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=8D=87=E7=BA=A7=E6=AD=A5=E9=AA=A4=E5=8F=98?= =?UTF-8?q?=E6=9B=B4=EF=BC=9A=E5=85=88=E9=87=8D=E5=91=BD=E5=90=8D=E6=97=A7?= =?UTF-8?q?=E8=A1=A8=E5=92=8C=E5=88=9D=E5=A7=8B=E5=8C=96=E6=96=B0=E8=A1=A8?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../01_create_table_2402.sql | 1995 ----------------- .../{03_rename_table.sql => 01_rename_old_table.sql} | 22 - .../{04_init_new_table.sql => 02_init_new_table.sql} | 0 .../{05_check.sql => 03_check.sql} | 0 ..._to_2402_view.sql => 04_create_table_2307_to_2402_view.sql} | 42 +- .../05_drop_table_2307_to_2402_view.sql | 10 + .../TSG 2307版本到2402版本升级操作/ck表升级步骤.md | 74 +- 7 files changed, 75 insertions(+), 2068 deletions(-) delete mode 100644 Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_create_table_2402.sql rename Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/{03_rename_table.sql => 01_rename_old_table.sql} (71%) rename Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/{04_init_new_table.sql => 02_init_new_table.sql} (100%) rename Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/{05_check.sql => 03_check.sql} (100%) rename Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/{02_create_table_2307_to_2402_view.sql => 04_create_table_2307_to_2402_view.sql} (96%) create mode 100644 Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/05_drop_table_2307_to_2402_view.sql diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_create_table_2402.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_create_table_2402.sql deleted file mode 100644 index 4862613..0000000 --- a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_create_table_2402.sql +++ /dev/null @@ -1,1995 +0,0 @@ -set distributed_ddl_task_timeout = 1800; - --- 创建七个2402临时表 - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local_2402 ON CLUSTER ck_cluster( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -ENGINE=MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local_2402 ON CLUSTER ck_cluster( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -ENGINE=MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local_2402 ON CLUSTER ck_cluster( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -ENGINE=MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local_2402 ON CLUSTER ck_cluster( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - ingestion_time Int64, - processing_time Int64, - insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - address_type Int32, - vsys_id Int32, - client_ip String, - client_port Int32, - server_ip String, - server_port Int32, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_eml_file String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String -) -ENGINE=MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id, session_id, recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local_2402 ON CLUSTER ck_cluster ( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String,  - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local_2402 ON CLUSTER ck_cluster ( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String,  - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - doh_url String, - doh_host String, - doh_request_line String, - doh_response_line String, - doh_cookie String, - doh_referer String, - doh_user_agent String, - doh_content_length String, - doh_content_type String, - doh_set_cookie String, - doh_version String, - doh_message_id Int64, - doh_qr Nullable(Int64), - doh_opcode Nullable(Int64), - doh_aa Nullable(Int64), - doh_tc Nullable(Int64), - doh_rd Nullable(Int64), - doh_ra Nullable(Int64), - doh_rcode Nullable(Int64), - doh_qdcount Nullable(Int64), - doh_ancount Nullable(Int64), - doh_nscount Nullable(Int64), - doh_arcount Nullable(Int64), - doh_qname String, - doh_qtype Nullable(Int64), - doh_qclass Nullable(Int64), - doh_cname String, - doh_sub Nullable(Int64), - doh_rr String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time); - -CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local_2402 ON CLUSTER ck_cluster ( - vsys_id Int32, - recv_time Int64, - log_id UInt64, - profile_id Int64, - start_time Int64, - end_time Int64, - attack_type String, - severity String, - conditions String, - destination_ip String, - destination_country String, - source_ip_list String, - source_country_list String, - session_rate Int64, - packet_rate Int64, - bit_rate Int64 -) -ENGINE = MergeTree -PARTITION BY toYYYYMMDD(toDate(recv_time)) -ORDER BY (vsys_id,destination_ip,recv_time,log_id); - --- 创建2402临时表物化视图:同步security_event和monitor_event表 --- tsg_galaxy_v3.security_event_materialized_view -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view_2402 on cluster ck_cluster -TO tsg_galaxy_v3.security_event_local_2402 -( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -AS -SELECT - recv_time, - log_id, - decoded_as, - session_id, - start_timestamp_ms, - end_timestamp_ms, - duration_ms, - tcp_handshake_latency_ms, - ingestion_time, - processing_time, - -- insert_time, - device_id, - out_link_id, - in_link_id, - device_tag, - data_center, - device_group, - sled_ip, - address_type, - vsys_id, - t_vsys_id, - flags, - flags_identify_info, - security_rule_list, - security_action, - monitor_rule_list, - shaping_rule_list, - proxy_rule_list, - statistics_rule_list, - sc_rule_list, - sc_rsp_raw, - sc_rsp_decrypted, - proxy_action, - proxy_pinning_status, - proxy_intercept_status, - proxy_passthrough_reason, - proxy_client_side_latency_ms, - proxy_server_side_latency_ms, - proxy_client_side_version, - proxy_server_side_version, - proxy_cert_verify, - proxy_intercept_error, - monitor_mirrored_pkts, - monitor_mirrored_bytes, - client_ip, - client_port, - client_os_desc, - client_geolocation, - client_asn, - subscriber_id, - imei, - imsi, - phone_number, - apn, - server_ip, - server_port, - server_os_desc, - server_geolocation, - server_asn, - server_fqdn, - server_domain, - app_transition, - app, - app_debug_info, - app_content, - fqdn_category_list, - ip_protocol, - decoded_path, - dns_message_id, - dns_qr, - dns_opcode, - dns_aa, - dns_tc, - dns_rd, - dns_ra, - dns_rcode, - dns_qdcount, - dns_ancount, - dns_nscount, - dns_arcount, - dns_qname, - dns_qtype, - dns_qclass, - dns_cname, - dns_sub, - dns_rr, - dns_response_latency_ms, - http_url, - http_host, - http_request_line, - http_response_line, - http_request_body, - http_response_body, - http_proxy_flag, - http_sequence, - http_cookie, - http_referer, - http_user_agent, - http_request_content_length, - http_request_content_type, - http_response_content_length, - http_response_content_type, - http_set_cookie, - http_version, - http_status_code, - http_response_latency_ms, - http_session_duration_ms, - http_action_file_size, - ssl_version, - ssl_sni, - ssl_san, - ssl_cn, - ssl_handshake_latency_ms, - ssl_ja3_hash, - ssl_ja3s_hash, - ssl_cert_issuer, - ssl_cert_subject, - ssl_esni_flag, - ssl_ech_flag, - dtls_cookie, - dtls_version, - dtls_sni, - dtls_san, - dtls_cn, - dtls_handshake_latency_ms, - dtls_ja3_fingerprint, - dtls_ja3_hash, - dtls_cert_issuer, - dtls_cert_subject, - mail_protocol_type, - mail_account, - mail_from_cmd, - mail_to_cmd, - mail_from, - mail_password, - mail_to, - mail_cc, - mail_bcc, - mail_subject, - mail_subject_charset, - mail_attachment_name, - mail_attachment_name_charset, - mail_eml_file, - ftp_account, - ftp_url, - ftp_link_type, - quic_version, - quic_sni, - quic_user_agent, - rdp_cookie, - rdp_security_protocol, - rdp_client_channels, - rdp_keyboard_layout, - rdp_client_version, - rdp_client_name, - rdp_client_product_id, - rdp_desktop_width, - rdp_desktop_height, - rdp_requested_color_depth, - rdp_certificate_type, - rdp_certificate_count, - rdp_certificate_permanent, - rdp_encryption_level, - rdp_encryption_method, - ssh_version, - ssh_auth_success, - ssh_client_version, - ssh_server_version, - ssh_cipher_alg, - ssh_mac_alg, - ssh_compression_alg, - ssh_kex_alg, - ssh_host_key_alg, - ssh_host_key, - ssh_hassh, - sip_call_id, - sip_originator_description, - sip_responder_description, - sip_user_agent, - sip_server, - sip_originator_sdp_connect_ip, - sip_originator_sdp_media_port, - sip_originator_sdp_media_type, - sip_originator_sdp_content, - sip_responder_sdp_connect_ip, - sip_responder_sdp_media_port, - sip_responder_sdp_media_type, - sip_responder_sdp_content, - sip_duration_s, - sip_bye, - rtp_payload_type_c2s, - rtp_payload_type_s2c, - rtp_pcap_path, - rtp_originator_dir, - stratum_cryptocurrency, - stratum_mining_pools, - stratum_mining_program, - stratum_mining_subscribe, - sent_pkts, - received_pkts, - sent_bytes, - received_bytes, - tcp_c2s_ip_fragments, - tcp_s2c_ip_fragments, - tcp_c2s_lost_bytes, - tcp_s2c_lost_bytes, - tcp_c2s_o3_pkts, - tcp_s2c_o3_pkts, - tcp_c2s_rtx_pkts, - tcp_s2c_rtx_pkts, - tcp_c2s_rtx_bytes, - tcp_s2c_rtx_bytes, - tcp_rtt_ms, - tcp_client_isn, - tcp_server_isn, - packet_capture_file, - in_src_mac, - out_src_mac, - in_dest_mac, - out_dest_mac, - encapsulation, - dup_traffic_flag, - tunnel_endpoint_a_desc, - tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.session_record_local_2402 -WHERE empty(security_rule_list) = 0 -; - --- tsg_galaxy_v3.monitor_event_materialized_view -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view_2402 on cluster ck_cluster -TO tsg_galaxy_v3.monitor_event_local_2402 -( - recv_time Int64, - log_id UInt64, - decoded_as String, - session_id UInt64, - start_timestamp_ms DateTime64(3), - end_timestamp_ms DateTime64(3), - duration_ms Int32, - tcp_handshake_latency_ms Nullable(Int32), - ingestion_time Int64, - processing_time Int64, - -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), - device_id String, - out_link_id Nullable(Int32), - in_link_id Nullable(Int32), - device_tag String, - data_center String, - device_group String, - sled_ip String, - address_type Int32, - vsys_id Int32, - t_vsys_id Int32, - flags Int64, - flags_identify_info String, - security_rule_list Array(Int64), - security_action String, - monitor_rule_list Array(Int64), - shaping_rule_list Array(Int64), - proxy_rule_list Array(Int64), - statistics_rule_list Array(Int64), - sc_rule_list Array(Int64), - sc_rsp_raw Array(Int64), - sc_rsp_decrypted Array(Int64), - proxy_action String, - proxy_pinning_status Nullable(Int32), - proxy_intercept_status Nullable(Int32), - proxy_passthrough_reason String, - proxy_client_side_latency_ms Nullable(Int32), - proxy_server_side_latency_ms Nullable(Int32), - proxy_client_side_version String, - proxy_server_side_version String, - proxy_cert_verify Nullable(Int32), - proxy_intercept_error String, - monitor_mirrored_pkts Nullable(Int32), - monitor_mirrored_bytes Nullable(Int32), - client_ip String, - client_port Int32, - client_os_desc String, - client_geolocation LowCardinality(String), - client_asn Nullable(Int64), - subscriber_id String, - imei String, - imsi String, - phone_number String, - apn String, - server_ip String, - server_port Int32, - server_os_desc String, - server_geolocation LowCardinality(String), - server_asn Nullable(Int64), - server_fqdn String, - server_domain String, - app_transition String, - app LowCardinality(String), - app_debug_info String, - app_content String, - fqdn_category_list Array(Int64), - ip_protocol LowCardinality(String), - decoded_path LowCardinality(String), - dns_message_id Nullable(Int32), - dns_qr Nullable(Int32), - dns_opcode Nullable(Int32), - dns_aa Nullable(Int32), - dns_tc Nullable(Int32), - dns_rd Nullable(Int32), - dns_ra Nullable(Int32), - dns_rcode Nullable(Int32), - dns_qdcount Nullable(Int32), - dns_ancount Nullable(Int32), - dns_nscount Nullable(Int32), - dns_arcount Nullable(Int32), - dns_qname String, - dns_qtype Nullable(Int32), - dns_qclass Nullable(Int32), - dns_cname String, - dns_sub Nullable(Int32), - dns_rr String, - dns_response_latency_ms Nullable(Int32), - http_url String, - http_host String, - http_request_line String, - http_response_line String, - http_request_body String, - http_response_body String, - http_proxy_flag Nullable(Int32), - http_sequence Nullable(Int32), - http_cookie String, - http_referer String, - http_user_agent String, - http_request_content_length Nullable(Int64), - http_request_content_type String, - http_response_content_length Nullable(Int64), - http_response_content_type String, - http_set_cookie String, - http_version String, - http_status_code Nullable(Int32), - http_response_latency_ms Nullable(Int32), - http_session_duration_ms Nullable(Int32), - http_action_file_size Nullable(Int64), - ssl_version String, - ssl_sni String, - ssl_san String, - ssl_cn String, - ssl_handshake_latency_ms Nullable(Int32), - ssl_ja3_hash String, - ssl_ja3s_hash String, - ssl_cert_issuer String, - ssl_cert_subject String, - ssl_esni_flag Nullable(Int32), - ssl_ech_flag Nullable(Int32), - dtls_cookie String, - dtls_version String, - dtls_sni String, - dtls_san String, - dtls_cn String, - dtls_handshake_latency_ms Nullable(Int32), - dtls_ja3_fingerprint String, - dtls_ja3_hash String, - dtls_cert_issuer String, - dtls_cert_subject String, - mail_protocol_type String, - mail_account String, - mail_from_cmd String, - mail_to_cmd String, - mail_from String, - mail_password String, - mail_to String, - mail_cc String, - mail_bcc String, - mail_subject String, - mail_subject_charset String, - mail_attachment_name String, - mail_attachment_name_charset String, - mail_eml_file String, - ftp_account String, - ftp_url String, - ftp_link_type String, - quic_version String, - quic_sni String, - quic_user_agent String, - rdp_cookie String, - rdp_security_protocol String, - rdp_client_channels String, - rdp_keyboard_layout String, - rdp_client_version String, - rdp_client_name String, - rdp_client_product_id String, - rdp_desktop_width String, - rdp_desktop_height String, - rdp_requested_color_depth String, - rdp_certificate_type String, - rdp_certificate_count Nullable(Int32), - rdp_certificate_permanent Nullable(Int32), - rdp_encryption_level String, - rdp_encryption_method String, - ssh_version String, - ssh_auth_success String, - ssh_client_version String, - ssh_server_version String, - ssh_cipher_alg String, - ssh_mac_alg String, - ssh_compression_alg String, - ssh_kex_alg String, - ssh_host_key_alg String, - ssh_host_key String, - ssh_hassh String, - sip_call_id String, - sip_originator_description String, - sip_responder_description String, - sip_user_agent String, - sip_server String, - sip_originator_sdp_connect_ip String, - sip_originator_sdp_media_port Nullable(Int32), - sip_originator_sdp_media_type String, - sip_originator_sdp_content String, - sip_responder_sdp_connect_ip String, - sip_responder_sdp_media_port Nullable(Int32), - sip_responder_sdp_media_type String, - sip_responder_sdp_content String, - sip_duration_s Nullable(Int32), - sip_bye String, - rtp_payload_type_c2s Nullable(Int32), - rtp_payload_type_s2c Nullable(Int32), - rtp_pcap_path String, - rtp_originator_dir Nullable(Int32), - stratum_cryptocurrency String, - stratum_mining_pools String, - stratum_mining_program String, - stratum_mining_subscribe String, - sent_pkts Int64, - received_pkts Int64, - sent_bytes Int64, - received_bytes Int64, - tcp_c2s_ip_fragments Nullable(Int64), - tcp_s2c_ip_fragments Nullable(Int64), - tcp_c2s_lost_bytes Nullable(Int64), - tcp_s2c_lost_bytes Nullable(Int64), - tcp_c2s_o3_pkts Nullable(Int64), - tcp_s2c_o3_pkts Nullable(Int64), - tcp_c2s_rtx_pkts Nullable(Int64), - tcp_s2c_rtx_pkts Nullable(Int64), - tcp_c2s_rtx_bytes Nullable(Int64), - tcp_s2c_rtx_bytes Nullable(Int64), - tcp_rtt_ms Nullable(Int32), - tcp_client_isn Nullable(Int64), - tcp_server_isn Nullable(Int64), - packet_capture_file String, - in_src_mac String, - out_src_mac String, - in_dest_mac String, - out_dest_mac String, - encapsulation String, - dup_traffic_flag Nullable(Int32), - tunnel_endpoint_a_desc String, - tunnel_endpoint_b_desc String -) -AS -SELECT - recv_time, - log_id, - decoded_as, - session_id, - start_timestamp_ms, - end_timestamp_ms, - duration_ms, - tcp_handshake_latency_ms, - ingestion_time, - processing_time, - -- insert_time, - device_id, - out_link_id, - in_link_id, - device_tag, - data_center, - device_group, - sled_ip, - address_type, - vsys_id, - t_vsys_id, - flags, - flags_identify_info, - security_rule_list, - security_action, - monitor_rule_list, - shaping_rule_list, - proxy_rule_list, - statistics_rule_list, - sc_rule_list, - sc_rsp_raw, - sc_rsp_decrypted, - proxy_action, - proxy_pinning_status, - proxy_intercept_status, - proxy_passthrough_reason, - proxy_client_side_latency_ms, - proxy_server_side_latency_ms, - proxy_client_side_version, - proxy_server_side_version, - proxy_cert_verify, - proxy_intercept_error, - monitor_mirrored_pkts, - monitor_mirrored_bytes, - client_ip, - client_port, - client_os_desc, - client_geolocation, - client_asn, - subscriber_id, - imei, - imsi, - phone_number, - apn, - server_ip, - server_port, - server_os_desc, - server_geolocation, - server_asn, - server_fqdn, - server_domain, - app_transition, - app, - app_debug_info, - app_content, - fqdn_category_list, - ip_protocol, - decoded_path, - dns_message_id, - dns_qr, - dns_opcode, - dns_aa, - dns_tc, - dns_rd, - dns_ra, - dns_rcode, - dns_qdcount, - dns_ancount, - dns_nscount, - dns_arcount, - dns_qname, - dns_qtype, - dns_qclass, - dns_cname, - dns_sub, - dns_rr, - dns_response_latency_ms, - http_url, - http_host, - http_request_line, - http_response_line, - http_request_body, - http_response_body, - http_proxy_flag, - http_sequence, - http_cookie, - http_referer, - http_user_agent, - http_request_content_length, - http_request_content_type, - http_response_content_length, - http_response_content_type, - http_set_cookie, - http_version, - http_status_code, - http_response_latency_ms, - http_session_duration_ms, - http_action_file_size, - ssl_version, - ssl_sni, - ssl_san, - ssl_cn, - ssl_handshake_latency_ms, - ssl_ja3_hash, - ssl_ja3s_hash, - ssl_cert_issuer, - ssl_cert_subject, - ssl_esni_flag, - ssl_ech_flag, - dtls_cookie, - dtls_version, - dtls_sni, - dtls_san, - dtls_cn, - dtls_handshake_latency_ms, - dtls_ja3_fingerprint, - dtls_ja3_hash, - dtls_cert_issuer, - dtls_cert_subject, - mail_protocol_type, - mail_account, - mail_from_cmd, - mail_to_cmd, - mail_from, - mail_password, - mail_to, - mail_cc, - mail_bcc, - mail_subject, - mail_subject_charset, - mail_attachment_name, - mail_attachment_name_charset, - mail_eml_file, - ftp_account, - ftp_url, - ftp_link_type, - quic_version, - quic_sni, - quic_user_agent, - rdp_cookie, - rdp_security_protocol, - rdp_client_channels, - rdp_keyboard_layout, - rdp_client_version, - rdp_client_name, - rdp_client_product_id, - rdp_desktop_width, - rdp_desktop_height, - rdp_requested_color_depth, - rdp_certificate_type, - rdp_certificate_count, - rdp_certificate_permanent, - rdp_encryption_level, - rdp_encryption_method, - ssh_version, - ssh_auth_success, - ssh_client_version, - ssh_server_version, - ssh_cipher_alg, - ssh_mac_alg, - ssh_compression_alg, - ssh_kex_alg, - ssh_host_key_alg, - ssh_host_key, - ssh_hassh, - sip_call_id, - sip_originator_description, - sip_responder_description, - sip_user_agent, - sip_server, - sip_originator_sdp_connect_ip, - sip_originator_sdp_media_port, - sip_originator_sdp_media_type, - sip_originator_sdp_content, - sip_responder_sdp_connect_ip, - sip_responder_sdp_media_port, - sip_responder_sdp_media_type, - sip_responder_sdp_content, - sip_duration_s, - sip_bye, - rtp_payload_type_c2s, - rtp_payload_type_s2c, - rtp_pcap_path, - rtp_originator_dir, - stratum_cryptocurrency, - stratum_mining_pools, - stratum_mining_program, - stratum_mining_subscribe, - sent_pkts, - received_pkts, - sent_bytes, - received_bytes, - tcp_c2s_ip_fragments, - tcp_s2c_ip_fragments, - tcp_c2s_lost_bytes, - tcp_s2c_lost_bytes, - tcp_c2s_o3_pkts, - tcp_s2c_o3_pkts, - tcp_c2s_rtx_pkts, - tcp_s2c_rtx_pkts, - tcp_c2s_rtx_bytes, - tcp_s2c_rtx_bytes, - tcp_rtt_ms, - tcp_client_isn, - tcp_server_isn, - packet_capture_file, - in_src_mac, - out_src_mac, - in_dest_mac, - out_dest_mac, - encapsulation, - dup_traffic_flag, - tunnel_endpoint_a_desc, - tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.session_record_local_2402 -WHERE empty(monitor_rule_list) = 0 -; diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_rename_table.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql similarity index 71% rename from Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_rename_table.sql rename to Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql index f4f4198..99e55ba 100644 --- a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_rename_table.sql +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql @@ -1,14 +1,5 @@ set distributed_ddl_task_timeout = 180; --- 删除源表同步到临时表物化视图, 七个表 -drop view if exists tsg_galaxy_v3.session_record_local_2307_to_2402_view on cluster ck_cluster; -drop view if exists tsg_galaxy_v3.security_event_local_2307_to_security_event_local_2402_view on cluster ck_cluster; -drop view if exists tsg_galaxy_v3.security_event_local_2307_to_monitor_event_local_2402_view on cluster ck_cluster; -drop view if exists tsg_galaxy_v3.transaction_record_local_2307_to_2402_view on cluster ck_cluster; -drop view if exists tsg_galaxy_v3.voip_record_local_2307_to_2402_view on cluster ck_cluster; -drop view if exists tsg_galaxy_v3.proxy_event_local_2307_to_2402_view on cluster ck_cluster; -drop view if exists tsg_galaxy_v3.dos_event_local_2307_to_2402_view on cluster ck_cluster; - -- 删除源表同步子表物化视图 drop VIEW IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster; drop VIEW IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster; @@ -81,16 +72,3 @@ drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query; drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster; drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query; drop table IF EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster; - --- 删除临时表之间物化视图 -drop VIEW IF EXISTS tsg_galaxy_v3.security_event_materialized_view_2402 ON CLUSTER ck_cluster; -drop VIEW IF EXISTS tsg_galaxy_v3.monitor_event_materialized_view_2402 ON CLUSTER ck_cluster; - --- 临时表rename到目标表 -RENAME TABLE tsg_galaxy_v3.session_record_local_2402 to tsg_galaxy_v3.session_record_local on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.security_event_local_2402 to tsg_galaxy_v3.security_event_local on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.monitor_event_local_2402 to tsg_galaxy_v3.monitor_event_local on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.transaction_record_local_2402 to tsg_galaxy_v3.transaction_record_local on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.voip_record_local_2402 to tsg_galaxy_v3.voip_record_local on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.proxy_event_local_2402 to tsg_galaxy_v3.proxy_event_local on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.dos_event_local_2402 to tsg_galaxy_v3.dos_event_local on cluster ck_cluster; diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/04_init_new_table.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_init_new_table.sql similarity index 100% rename from Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/04_init_new_table.sql rename to Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_init_new_table.sql diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/05_check.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_check.sql similarity index 100% rename from Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/05_check.sql rename to Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_check.sql diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_create_table_2307_to_2402_view.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/04_create_table_2307_to_2402_view.sql similarity index 96% rename from Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_create_table_2307_to_2402_view.sql rename to Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/04_create_table_2307_to_2402_view.sql index 10e0899..947e80f 100644 --- a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_create_table_2307_to_2402_view.sql +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/04_create_table_2307_to_2402_view.sql @@ -2,9 +2,9 @@ set distributed_ddl_task_timeout = 180; -- 创建同步源表同步2402临时表物化视图, 七个表 --- tsg_galaxy_v3.session_record_local_2402 +-- tsg_galaxy_v3.session_record_local CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.session_record_local_2307_to_2402_view on cluster ck_cluster -TO tsg_galaxy_v3.session_record_local_2402 +TO tsg_galaxy_v3.session_record_local ( recv_time Int64, log_id UInt64, @@ -457,12 +457,12 @@ SELECT common_has_dup_traffic as dup_traffic_flag, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.session_record_local +FROM tsg_galaxy_v3.session_record_local_old ; --- tsg_galaxy_v3.security_event_local_2402 +-- tsg_galaxy_v3.security_event_local CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_local_2307_to_security_event_local_2402_view on cluster ck_cluster -TO tsg_galaxy_v3.security_event_local_2402 +TO tsg_galaxy_v3.security_event_local ( recv_time Int64, log_id UInt64, @@ -919,13 +919,13 @@ SELECT common_has_dup_traffic as dup_traffic_flag, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.security_event_local +FROM tsg_galaxy_v3.security_event_local_old where common_action in (16, 96) ; --- tsg_galaxy_v3.monitor_event_local_2402 +-- tsg_galaxy_v3.monitor_event_local CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_local_2307_to_monitor_event_local_2402_view on cluster ck_cluster -TO tsg_galaxy_v3.monitor_event_local_2402 +TO tsg_galaxy_v3.monitor_event_local ( recv_time Int64, log_id UInt64, @@ -1382,13 +1382,13 @@ SELECT common_has_dup_traffic as dup_traffic_flag, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.security_event_local +FROM tsg_galaxy_v3.security_event_local_old where common_action = 1 ; --- tsg_galaxy_v3.transaction_record_local_2402 +-- tsg_galaxy_v3.transaction_record_local CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.transaction_record_local_2307_to_2402_view on cluster ck_cluster -TO tsg_galaxy_v3.transaction_record_local_2402 +TO tsg_galaxy_v3.transaction_record_local ( recv_time Int64, log_id UInt64, @@ -1565,12 +1565,12 @@ SELECT sip_responder_sdp_content as sip_responder_sdp_content, sip_duration_s as sip_duration_s, sip_bye as sip_bye -FROM tsg_galaxy_v3.transaction_record_local +FROM tsg_galaxy_v3.transaction_record_local_old ; --- tsg_galaxy_v3.voip_record_local_2402 +-- tsg_galaxy_v3.voip_record_local CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.voip_record_local_2307_to_2402_view on cluster ck_cluster -TO tsg_galaxy_v3.voip_record_local_2402 +TO tsg_galaxy_v3.voip_record_local ( recv_time Int64, log_id UInt64, @@ -1801,12 +1801,12 @@ SELECT common_has_dup_traffic as dup_traffic_flag, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.voip_record_local +FROM tsg_galaxy_v3.voip_record_local_old ; --- tsg_galaxy_v3.proxy_event_local_2402 +-- tsg_galaxy_v3.proxy_event_local CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.proxy_event_local_2307_to_2402_view on cluster ck_cluster -TO tsg_galaxy_v3.proxy_event_local_2402 +TO tsg_galaxy_v3.proxy_event_local ( recv_time Int64, log_id UInt64, @@ -2099,12 +2099,12 @@ SELECT common_has_dup_traffic as dup_traffic_flag, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc -FROM tsg_galaxy_v3.proxy_event_local +FROM tsg_galaxy_v3.proxy_event_local_old ; --- tsg_galaxy_v3.dos_event_local_2402 +-- tsg_galaxy_v3.dos_event_local CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.dos_event_local_2307_to_2402_view on cluster ck_cluster -TO tsg_galaxy_v3.dos_event_local_2402 +TO tsg_galaxy_v3.dos_event_local ( vsys_id Int32, recv_time Int64, @@ -2141,5 +2141,5 @@ SELECT session_rate as session_rate, packet_rate as packet_rate, bit_rate as bit_rate -FROM tsg_galaxy_v3.dos_event_local +FROM tsg_galaxy_v3.dos_event_local_old ; \ No newline at end of file diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/05_drop_table_2307_to_2402_view.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/05_drop_table_2307_to_2402_view.sql new file mode 100644 index 0000000..9afb690 --- /dev/null +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/05_drop_table_2307_to_2402_view.sql @@ -0,0 +1,10 @@ +set distributed_ddl_task_timeout = 180; + +-- 删除旧表同步新表物化视图, 七个表 +drop view if exists tsg_galaxy_v3.session_record_local_2307_to_2402_view on cluster ck_cluster; +drop view if exists tsg_galaxy_v3.security_event_local_2307_to_security_event_local_2402_view on cluster ck_cluster; +drop view if exists tsg_galaxy_v3.security_event_local_2307_to_monitor_event_local_2402_view on cluster ck_cluster; +drop view if exists tsg_galaxy_v3.transaction_record_local_2307_to_2402_view on cluster ck_cluster; +drop view if exists tsg_galaxy_v3.voip_record_local_2307_to_2402_view on cluster ck_cluster; +drop view if exists tsg_galaxy_v3.proxy_event_local_2307_to_2402_view on cluster ck_cluster; +drop view if exists tsg_galaxy_v3.dos_event_local_2307_to_2402_view on cluster ck_cluster; diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md index 8c0d292..594df1c 100644 --- a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md @@ -6,50 +6,64 @@ clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1" 若返回结果为空则可执行升级步骤，否则需要等待。 -# 一、实时同步任务 +# 一、停止旧表ck入库任务 -* 1.创建临时表 -```sh -clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_create_table_2402.sql -``` +停止旧表ck入库任务 -* 2.创建源表同步到临时表的物化视图 -```sh -clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 02_create_table_2307_to_2402_view.sql -``` +# 二、旧表重命名为历史表 -# 二、升级各个数据中心(可选) - -* 1.国家中心启动ck入库任务(XX_2402 task)同步临时表：创建kafka临时topic(以_2402结尾)，启动ck同步到临时表任务 - -* 2.升级各个分数据中心：启动ETL任务发送到国家中心临时topic(以_2402结尾) - -# 三、所有分中心升级完毕，临时表切换为目标表，源表切换为历史表 - -* 1.停止源表ck入库任务 - -* 2.停止ck入库临时表任务 - -* 3.重命名旧表和临时表 +重命名旧表, 删除废弃表 ```sql -clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 03_rename_table.sql +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_rename_old_table.sql ``` -* 4.执行2402版本初始化建表语句 +# 三、初始化新表 + +* 1.执行2402版本初始化建表语句 ``` -clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 04_init_new_table.sql +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 02_init_new_table.sql ``` -* 5.校验表结构 +* 2.校验表结构 ``` -clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 05_check.sql +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 03_check.sql ``` -无报错信息说明校验通过 -* 6.启动目标表ck入库任务(升级完成) +**无报错信息说明校验通过** + +# 四、创建旧表同步新表任务(可选) + +创建旧表同步到新表的物化视图(如果还有分数据中心向旧表写数据) +```sh +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 04_create_table_2307_to_2402_view.sql +``` + +# 五、启动ck入库任务 + +* 1.启动新表ck入库任务 + +* 2.启动旧表ck入库任务(如果还有分数据中心向旧表写数据) +```sh +# 重命名旧表, 删除废弃表后, 存在的旧表: +tsg_galaxy_v3.session_record_local_old +tsg_galaxy_v3.security_event_local_old +tsg_galaxy_v3.transaction_record_local_old +tsg_galaxy_v3.voip_record_local_old +tsg_galaxy_v3.proxy_event_local_old +tsg_galaxy_v3.dos_event_local_old +``` + +# 六、各个数据中心全部升级完成后停止旧表ck入库任务 + +* 1.升级各个数据中心，各个数据中心全部升级完成后，停止旧表ck入库任务(如果启动的话) + +* 2.删除旧表同步新表物化视图 +```sh +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 05_drop_table_2307_to_2402_view.sql +``` -# 四、离线同步历史数据(可选) +# 七、离线同步历史数据(可选) 在query节点执行以下步骤，iplist.txt中为ck所有data节点ip地址。