modify ddl and configuration template directory
This commit is contained in:
doufenghu
@@ -0,0 +1,11 @@
|
||||
select 'session_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.session_record_old;
|
||||
|
||||
select 'security_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.security_event_old;
|
||||
|
||||
select 'transaction_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.transaction_record_old;
|
||||
|
||||
select 'voip_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.voip_record_old;
|
||||
|
||||
select 'proxy_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.proxy_event_old;
|
||||
|
||||
select 'dos_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.dos_event_old;
|
||||
@@ -0,0 +1,107 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- 删除源表同步子表物化视图
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster;
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster;
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster;
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster;
|
||||
|
||||
-- 删除源表子表相关回表
|
||||
drop table IF EXISTS tsg_galaxy_v3.interim_session_record_local ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_client_ip_local ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_ip_local ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLUSTER ck_cluster;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_cluster;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_query;
|
||||
|
||||
-- 源表rename到历史表
|
||||
RENAME TABLE tsg_galaxy_v3.session_record_local to tsg_galaxy_v3.session_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.security_event_local to tsg_galaxy_v3.security_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.transaction_record_local to tsg_galaxy_v3.transaction_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.voip_record_local to tsg_galaxy_v3.voip_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.proxy_event_local to tsg_galaxy_v3.proxy_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.dos_event_local to tsg_galaxy_v3.dos_event_local_old on cluster ck_cluster;
|
||||
|
||||
-- 删除源表分布式表
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.dos_event ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.dos_event ON CLUSTER ck_cluster;
|
||||
|
||||
-- assessment_event不用迁移
|
||||
drop table IF EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster;
|
||||
|
||||
-- 删除废弃表
|
||||
drop table IF EXISTS tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.gtpc_record on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.gtpc_record on cluster ck_query;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_onff_log_local on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_onff_log on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_onff_log on cluster ck_query;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_record_local on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_record on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_record on cluster ck_query;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster;
|
||||
|
||||
-- 创建源码分布式表old
|
||||
create table IF NOT EXISTS tsg_galaxy_v3.session_record_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local_old,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_v3.security_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local_old,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_v3.proxy_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_old ON CLUSTER ck_query(
|
||||
log_id UInt64,
|
||||
profile_id UInt64,
|
||||
start_time Int64
|
||||
) ENGINE = Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local_old,rand());
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,20 @@
|
||||
SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
|
||||
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT vsys_id, recv_time, log_id, profile_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, session_rate, packet_rate, bit_rate
|
||||
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
|
||||
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,10 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- 删除旧表同步新表物化视图, 七个表
|
||||
drop view if exists tsg_galaxy_v3.session_record_local_2307_to_2402_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.security_event_local_2307_to_security_event_local_2402_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.security_event_local_2307_to_monitor_event_local_2402_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.transaction_record_local_2307_to_2402_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.voip_record_local_2307_to_2402_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.proxy_event_local_2307_to_2402_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.dos_event_local_2307_to_2402_view on cluster ck_cluster;
|
||||
@@ -0,0 +1,17 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- 删除旧表
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_local_old on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.security_event_local_old on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.transaction_record_local_old on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.voip_record_local_old on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.proxy_event_local_old on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.dos_event_local_old on cluster ck_cluster;
|
||||
|
||||
-- 删除旧表分布式表
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.session_record_old ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.security_event_old ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.transaction_record_old ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.voip_record_old ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.proxy_event_old ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.dos_event_old ON CLUSTER ck_query;
|
||||
@@ -0,0 +1,314 @@
|
||||
由于需要升级多个数据中心，为了保证业务连续性且不丢失数据，采用实时增量同步的方法进行数据迁移。
|
||||
具体步骤：
|
||||
- Step1 ：停止国家中心gohangout入库任务。
|
||||
- Step2 ：23.07版本clickhouse库表_local表重命名_local_old,删除相关视图及废弃表。
|
||||
- Step3 ：升级国家中心，初始化24.02版本clickhouse库表，并进行校验。
|
||||
- Step4 ：国家中心ck创建同步物化视图（*_local_old->*_local）。gohangout入库任务修改配置将入库表修改为*_local_old,已经删除的库表对应的gohangout任务也可删除，gohangout保留任务（session_record,security_event,transaction_record,voip_record,proxy_event,dos_event）,重启gohangout。
|
||||
- Step5 : 单个分中心升级，TSG OS → 分中心Kafka → 分中心ETL（grootstream）→ 国家中心Kafka（*-PROCESSED） → 国家中心Groot → 24.02库表
|
||||
- 未升级的分中心仍保留原先的ETL任务，最终汇入国家中心kafka(*-COMPLETED)中→ 国家中心（gohangout） → *_local_old库表 -> ck物化视图同步至24.02库表中
|
||||
- Step6 : 所有分中心升级完毕，关闭国家中心gohangout，删除ck同步物化视图。
|
||||
- Step7 : 按照具体情况选择是否删除所有的历史表 *_local_old，或是否启动离线同步历史数据任务。
|
||||
|
||||
|
||||
# 说明
|
||||
* 请按步骤依次执行，执行脚本报错时联系研发处理后再执行之后的步骤。
|
||||
* 所有ck步骤都需要在query节点执行
|
||||
* 执行所有sql语句之前需要停止日志留存调度任务，确保ck中无分布式ddl语句H执行，否则执行的sql会阻塞住，影响后续步骤执行
|
||||
验证sql需要在query节点执行
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1"
|
||||
若返回结果为空则可执行升级步骤，否则需要等待。
|
||||
|
||||
# 一、停止旧表ck入库任务
|
||||
|
||||
停止旧表ck入库任务
|
||||
|
||||
# 二、旧表重命名为历史表
|
||||
|
||||
* 1.重命名旧表, 删除废弃表
|
||||
```sql
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_rename_old_table.sql
|
||||
```
|
||||
|
||||
* 2.查看旧表数据量
|
||||
```
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_cat_old_table_row_count.sql
|
||||
```
|
||||
|
||||
# 三、初始化新表
|
||||
|
||||
* 1.执行2402版本初始化建表语句
|
||||
```
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 02_init_new_table.sql
|
||||
```
|
||||
|
||||
* 2.校验表结构
|
||||
```
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 03_check.sql
|
||||
```
|
||||
|
||||
**无报错信息说明校验通过**
|
||||
|
||||
# 四、创建旧表同步新表任务(可选)
|
||||
|
||||
创建旧表同步到新表的物化视图(如果还有分数据中心向旧表写数据)
|
||||
```sh
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 04_create_table_2307_to_2402_view.sql
|
||||
```
|
||||
|
||||
# 五、启动ck入库任务
|
||||
|
||||
* 1.启动新表ck入库任务
|
||||
|
||||
* 2.启动旧表ck入库任务(如果还有分数据中心向旧表写数据)
|
||||
```sh
|
||||
# 重命名旧表, 删除废弃表后, 存在的旧表:
|
||||
tsg_galaxy_v3.session_record_local_old
|
||||
tsg_galaxy_v3.security_event_local_old
|
||||
tsg_galaxy_v3.transaction_record_local_old
|
||||
tsg_galaxy_v3.voip_record_local_old
|
||||
tsg_galaxy_v3.proxy_event_local_old
|
||||
tsg_galaxy_v3.dos_event_local_old
|
||||
```
|
||||
|
||||
* 3.查看旧表是否有数据入库(间隔一段时间查询,后面查询比前面查询行数多说明有数据写入)
|
||||
```
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_cat_old_table_row_count.sql
|
||||
```
|
||||
|
||||
# 六、各个数据中心全部升级完成后停止旧表ck入库任务
|
||||
|
||||
* 1.升级各个数据中心，各个数据中心全部升级完成后，停止旧表ck入库任务(如果启动的话)
|
||||
|
||||
* 2.删除旧表同步新表物化视图
|
||||
```sh
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 05_drop_table_2307_to_2402_view.sql
|
||||
```
|
||||
|
||||
|
||||
# 七、离线同步历史数据(可选)
|
||||
|
||||
在query节点执行以下步骤，iplist.txt中为ck所有data节点ip地址。
|
||||
|
||||
步骤描述：
|
||||
* 1.进入migrate_table_2402文件夹,使脚本可执行
|
||||
```
|
||||
chmod +x ./*.sh
|
||||
```
|
||||
|
||||
* 2.分发迁移脚本到data节点
|
||||
```
|
||||
./01_send_migrate_table_scripts.sh
|
||||
```
|
||||
|
||||
* 2.选择迁移某个表，同步需要时间区间的数据，时间区间:[实时同步任务开始时间向前推n天, 实时同步任务开始时间)，时间区间为左闭右开，不包含结束时间点。
|
||||
```
|
||||
# 迁移security_event表
|
||||
./02_start_migrate_table.sh security_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
```
|
||||
|
||||
* 3.监控data节点迁移情况，所有表迁移完成后，确认每个节点同步数据成功/失败批次数，如有失败批次确认是否需要处理
|
||||
```
|
||||
# 监控security_event表迁移
|
||||
./03_monitor_migrate_table.sh security_event
|
||||
```
|
||||
|
||||
* 4.选择下个张需要迁移的表，重复2-4步骤。支持选择迁移的表有: security_event, monitor_event, session_record, transaction_record, voip_record, proxy_event, dos_event。
|
||||
|
||||
|
||||
迁移和监控各个表执行命令示例：
|
||||
```sh
|
||||
# 迁移security_event表
|
||||
./02_start_migrate_table.sh security_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控security_event表迁移
|
||||
./03_monitor_migrate_table.sh security_event
|
||||
|
||||
|
||||
# 迁移monitor_event表
|
||||
./02_start_migrate_table.sh monitor_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控monitor_event表迁移
|
||||
./03_monitor_migrate_table.sh monitor_event
|
||||
|
||||
|
||||
# 迁移session_record表
|
||||
./02_start_migrate_table.sh session_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控session_record表迁移
|
||||
./03_monitor_migrate_table.sh session_record
|
||||
|
||||
|
||||
# 迁移transaction_record表
|
||||
./02_start_migrate_table.sh transaction_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控transaction_record表迁移
|
||||
./03_monitor_migrate_table.sh transaction_record
|
||||
|
||||
|
||||
# 迁移voip_record表
|
||||
./02_start_migrate_table.sh voip_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控voip_record表迁移
|
||||
./03_monitor_migrate_table.sh voip_record
|
||||
|
||||
|
||||
# 迁移proxy_event表
|
||||
./02_start_migrate_table.sh proxy_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控proxy_event表迁移
|
||||
./03_monitor_migrate_table.sh proxy_event
|
||||
|
||||
|
||||
# 迁移dos_event表
|
||||
./02_start_migrate_table.sh dos_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控dos_event表迁移
|
||||
./03_monitor_migrate_table.sh dos_event
|
||||
```
|
||||
|
||||
迁移日志无报错，数据迁移完成。
|
||||
|
||||
如果有数据迁移失败批次，查看新老表迁移数据量对应情况(ck每台**data**节点)：
|
||||
```sql
|
||||
-- security_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.security_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
and common_action in (16, 96)
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.security_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- monitor_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.security_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
and common_action = 1
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.monitor_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- session_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.session_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.session_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- transaction_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.transaction_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.transaction_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- voip_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.voip_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.voip_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- proxy_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.proxy_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.proxy_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- dos_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(start_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.dos_event_local_old
|
||||
WHERE start_time>= toUnixTimestamp('2024-01-10 00:00:00') and start_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(start_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(start_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.dos_event_local
|
||||
WHERE start_time >= toUnixTimestamp('2024-01-10 00:00:00') and start_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(start_time))
|
||||
order by d
|
||||
;
|
||||
```
|
||||
|
||||
# 八、删除旧表
|
||||
数据迁移完成后，不需要查看旧表时，删除旧表。
|
||||
|
||||
```sh
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 06_drop_old_table.sql
|
||||
```
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
# 遍历每个节点执行迁移
|
||||
for ip in `cat iplist.txt`
|
||||
do
|
||||
# 后台执行，输出日志
|
||||
echo "$ip节点"
|
||||
ssh $ip "[ ! -d $home ] && mkdir -p $home"
|
||||
scp -r $home/*local_table_to_2402.sh $ip:$home/
|
||||
ssh $ip "cd $home && chmod +x ./*.sh"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据，TSG24.01日志重组后数据迁移
|
||||
|
||||
# 迁移表 参数，可选值：session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00"
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00"
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数, 例如:240
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"session_record")
|
||||
# 迁移session_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"security_event")
|
||||
# 迁移security_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"monitor_event")
|
||||
# 迁移monitor_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"transaction_record")
|
||||
# 迁移transaction_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"voip_record")
|
||||
# 迁移voip_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"proxy_event")
|
||||
# 迁移proxy_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "slice_interval_minute参数必须是正确的分钟数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
# 遍历每个节点执行迁移
|
||||
for ip in `cat iplist.txt`
|
||||
do
|
||||
# 后台执行，输出日志
|
||||
echo "$ip 节点开始执行迁移"
|
||||
ssh $ip "cd $home && chmod +x ./*.sh && ./start_migrate_local_table_to_2402.sh $table '$data_start_time' '$data_end_time' $slice_interval_minute"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
table=$1
|
||||
if [ -z "$table" ]; then
|
||||
echo "缺少table参数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ips=($(cat iplist.txt))
|
||||
ips_size=${#ips[*]}
|
||||
ip_starts=$( seq 0 $(($ips_size - 1)) )
|
||||
ip_ends=$( seq 0 $(($ips_size - 1)) )
|
||||
|
||||
for ((i=0;i<$ips_size;i++))
|
||||
do
|
||||
ip_starts[$i]=0
|
||||
ip_ends[$i]=0
|
||||
done
|
||||
|
||||
while true ; do
|
||||
# 遍历每个节点
|
||||
for ((i=0;i<$ips_size;i++)); do
|
||||
ip=${ips[$i]}
|
||||
start=${ip_starts[$i]}
|
||||
end=${ip_ends[$i]}
|
||||
if [ $start -eq 0 ]; then
|
||||
info=$(ssh $ip "cat $home/log_$table.txt | grep migrate_table_start")
|
||||
if [ -n "$info" ]; then
|
||||
echo "${ip}迁移开始:${info}"
|
||||
ip_starts[$i]=1
|
||||
start=1
|
||||
fi
|
||||
fi
|
||||
if [ $start -eq 1 ] && [ $end -eq 0 ] ; then
|
||||
info=$(ssh $ip "cat $home/log_$table.txt | grep migrate_table_end")
|
||||
if [ -n "$info" ]; then
|
||||
echo "${ip}迁移结束:${info}"
|
||||
ip_ends[$i]=1
|
||||
end=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
#全部结束
|
||||
finish_cnt=0
|
||||
for ((i=0;i<$ips_size;i++)); do
|
||||
start=${ip_starts[$i]}
|
||||
end=${ip_ends[$i]}
|
||||
if [ $start -eq 1 ] && [ $end -eq 1 ] ; then
|
||||
finish_cnt=$(($finish_cnt+1))
|
||||
fi
|
||||
done
|
||||
if [ $finish_cnt -ge $ips_size ]; then
|
||||
echo "所有节点迁移结束"
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
@@ -0,0 +1 @@
|
||||
192.168.41.30
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据，TSG24.01日志重组后数据迁移
|
||||
|
||||
# 迁移表 参数，可选值：session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00"
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00"
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数, 例如:240
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"session_record")
|
||||
# 迁移session_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"security_event")
|
||||
# 迁移security_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"monitor_event")
|
||||
# 迁移monitor_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"transaction_record")
|
||||
# 迁移transaction_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"voip_record")
|
||||
# 迁移voip_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"proxy_event")
|
||||
# 迁移proxy_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "slice_interval_minute参数必须是正确的分钟数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 后台执行，输出日志
|
||||
nohup ./migrate_local_table_to_2402.sh "$table" "$data_start_time" "$data_end_time" $slice_interval_minute > "log_$table.txt" 2>&1 &
|
||||
echo "已启动迁移${table}表任务，时间范围[$data_start_time, $data_end_time], 每批迁移段分钟:$slice_interval_minute, 日志输出到:log_$table.txt。请查看日志文件确认每段数据迁移情况"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,96 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- 删除源表同步到临时表物化视图, 七个表
|
||||
drop view if exists tsg_galaxy_v3.session_record_local_2310_to_2401_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.security_event_local_2310_to_security_event_local_2401_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.security_event_local_2310_to_monitor_event_local_2401_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.transaction_record_local_2310_to_2401_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.voip_record_local_2310_to_2401_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.proxy_event_local_2310_to_2401_view on cluster ck_cluster;
|
||||
drop view if exists tsg_galaxy_v3.dos_event_local_2310_to_2401_view on cluster ck_cluster;
|
||||
|
||||
-- 删除源表同步子表物化视图
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster;
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster;
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster;
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster;
|
||||
|
||||
-- 删除源表子表相关回表
|
||||
drop table IF EXISTS tsg_galaxy_v3.interim_session_record_local ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_client_ip_local ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_ip_local ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLUSTER ck_cluster;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_cluster;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER ck_query;
|
||||
|
||||
-- 源表rename到历史表
|
||||
RENAME TABLE tsg_galaxy_v3.session_record_local to tsg_galaxy_v3.session_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.security_event_local to tsg_galaxy_v3.security_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.transaction_record_local to tsg_galaxy_v3.transaction_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.voip_record_local to tsg_galaxy_v3.voip_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.proxy_event_local to tsg_galaxy_v3.proxy_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.dos_event_local to tsg_galaxy_v3.dos_event_local_old on cluster ck_cluster;
|
||||
|
||||
-- 删除源表分布式表
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.dos_event ON CLUSTER ck_query;
|
||||
DROP TABLE IF EXISTS tsg_galaxy_v3.dos_event ON CLUSTER ck_cluster;
|
||||
|
||||
-- assessment_event不用迁移
|
||||
drop table IF EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster;
|
||||
|
||||
-- 删除废弃表
|
||||
drop table IF EXISTS tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.gtpc_record on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.gtpc_record on cluster ck_query;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_onff_log_local on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_onff_log on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_onff_log on cluster ck_query;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_record_local on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_record on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.radius_record on cluster ck_query;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query;
|
||||
|
||||
drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster;
|
||||
drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query;
|
||||
drop table IF EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster;
|
||||
|
||||
-- 删除临时表之间物化视图
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.security_event_materialized_view_2401 ON CLUSTER ck_cluster;
|
||||
drop VIEW IF EXISTS tsg_galaxy_v3.monitor_event_materialized_view_2401 ON CLUSTER ck_cluster;
|
||||
|
||||
-- 临时表rename到目标表
|
||||
RENAME TABLE tsg_galaxy_v3.session_record_local_2401 to tsg_galaxy_v3.session_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.security_event_local_2401 to tsg_galaxy_v3.security_event_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.monitor_event_local_2401 to tsg_galaxy_v3.monitor_event_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.transaction_record_local_2401 to tsg_galaxy_v3.transaction_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.voip_record_local_2401 to tsg_galaxy_v3.voip_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.proxy_event_local_2401 to tsg_galaxy_v3.proxy_event_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.dos_event_local_2401 to tsg_galaxy_v3.dos_event_local on cluster ck_cluster;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,20 @@
|
||||
SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
|
||||
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT vsys_id, recv_time, log_id, profile_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, session_rate, packet_rate, bit_rate
|
||||
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, tunnels, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, tunnels, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, tunnels, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, tunnels, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
|
||||
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, fqdn_category_list, ip_protocol, decoded_path, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, tunnels, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
# 说明
|
||||
* 请按步骤依次执行，执行脚本报错时联系研发处理后再执行之后的步骤。
|
||||
* 所有ck步骤都需要在query节点执行
|
||||
* 执行所有sql语句之前需要停止日志留存调度任务，确保ck中无分布式ddl语句H执行，否则执行的sql会阻塞住，影响后续步骤执行
|
||||
验证sql需要在query节点执行
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1"
|
||||
若返回结果为空则可执行升级步骤，否则需要等待。
|
||||
|
||||
# 一、实时同步任务
|
||||
|
||||
* 1.创建临时表
|
||||
```sh
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_create_table_2401.sql
|
||||
```
|
||||
|
||||
* 2.创建源表同步到临时表的物化视图
|
||||
```sh
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 02_create_table_2310_to_2401_view.sql
|
||||
```
|
||||
|
||||
# 二、升级各个数据中心(可选)
|
||||
|
||||
* 1.国家中心启动ck入库任务(XX_2401 task)同步临时表：创建kafka临时topic(以_2401结尾)，启动ck同步到临时表任务
|
||||
|
||||
* 2.升级各个分数据中心：启动ETL任务发送到国家中心临时topic(以_2401结尾)
|
||||
|
||||
# 三、所有分中心升级完毕，临时表切换为目标表，源表切换为历史表
|
||||
|
||||
* 1.停止源表ck入库任务
|
||||
|
||||
* 2.停止ck入库临时表任务
|
||||
|
||||
* 3.重命名旧表和临时表
|
||||
```sql
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 03_rename_table.sql
|
||||
```
|
||||
|
||||
* 4.执行2401版本初始化建表语句
|
||||
```
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 04_init_new_table.sql
|
||||
```
|
||||
|
||||
* 5.校验表结构
|
||||
```
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 05_check.sql
|
||||
```
|
||||
无报错信息说明校验通过
|
||||
|
||||
* 6.启动目标表ck入库任务(升级完成)
|
||||
|
||||
|
||||
# 四、离线同步历史数据(可选)
|
||||
|
||||
在query节点执行以下步骤，iplist.txt中为ck所有data节点ip地址。
|
||||
|
||||
步骤描述：
|
||||
* 1.进入migrate_table_2401文件夹,使脚本可执行
|
||||
```
|
||||
chmod +x ./*.sh
|
||||
```
|
||||
|
||||
* 2.分发迁移脚本到data节点
|
||||
```
|
||||
./01_send_migrate_table_scripts.sh
|
||||
```
|
||||
|
||||
* 2.选择迁移某个表，同步需要时间区间的数据，时间区间:[实时同步任务开始时间向前推n天, 实时同步任务开始时间)，时间区间为左闭右开，不包含结束时间点。
|
||||
```
|
||||
# 迁移security_event表
|
||||
./02_start_migrate_table.sh security_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
```
|
||||
|
||||
* 3.监控data节点迁移情况，所有表迁移完成后，确认每个节点同步数据成功/失败批次数，如有失败批次确认是否需要处理
|
||||
```
|
||||
# 监控security_event表迁移
|
||||
./03_monitor_migrate_table.sh security_event
|
||||
```
|
||||
|
||||
* 4.选择下个张需要迁移的表，重复2-4步骤。支持选择迁移的表有: security_event, monitor_event, session_record, transaction_record, voip_record, proxy_event, dos_event。
|
||||
|
||||
|
||||
迁移和监控各个表执行命令示例：
|
||||
```sh
|
||||
# 迁移security_event表
|
||||
./02_start_migrate_table.sh security_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控security_event表迁移
|
||||
./03_monitor_migrate_table.sh security_event
|
||||
|
||||
|
||||
# 迁移monitor_event表
|
||||
./02_start_migrate_table.sh monitor_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控monitor_event表迁移
|
||||
./03_monitor_migrate_table.sh monitor_event
|
||||
|
||||
|
||||
# 迁移session_record表
|
||||
./02_start_migrate_table.sh session_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控session_record表迁移
|
||||
./03_monitor_migrate_table.sh session_record
|
||||
|
||||
|
||||
# 迁移transaction_record表
|
||||
./02_start_migrate_table.sh transaction_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控transaction_record表迁移
|
||||
./03_monitor_migrate_table.sh transaction_record
|
||||
|
||||
|
||||
# 迁移voip_record表
|
||||
./02_start_migrate_table.sh voip_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控voip_record表迁移
|
||||
./03_monitor_migrate_table.sh voip_record
|
||||
|
||||
|
||||
# 迁移proxy_event表
|
||||
./02_start_migrate_table.sh proxy_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控proxy_event表迁移
|
||||
./03_monitor_migrate_table.sh proxy_event
|
||||
|
||||
|
||||
# 迁移dos_event表
|
||||
./02_start_migrate_table.sh dos_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控dos_event表迁移
|
||||
./03_monitor_migrate_table.sh dos_event
|
||||
```
|
||||
|
||||
迁移日志无报错，数据迁移完成。
|
||||
|
||||
如果有数据迁移失败批次，查看新老表迁移数据量对应情况(ck每台**data**节点)：
|
||||
```sql
|
||||
-- security_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.security_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
and common_action in (16, 96)
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.security_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- monitor_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.security_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
and common_action = 1
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.monitor_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- session_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.session_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.session_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- transaction_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.transaction_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.transaction_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- voip_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.voip_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.voip_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- proxy_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.proxy_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.proxy_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- dos_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(start_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.dos_event_local_old
|
||||
WHERE start_time>= toUnixTimestamp('2024-01-10 00:00:00') and start_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(start_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(start_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_v3.dos_event_local
|
||||
WHERE start_time >= toUnixTimestamp('2024-01-10 00:00:00') and start_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(start_time))
|
||||
order by d
|
||||
;
|
||||
```
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
# 遍历每个节点执行迁移
|
||||
for ip in `cat iplist.txt`
|
||||
do
|
||||
# 后台执行，输出日志
|
||||
echo "$ip节点"
|
||||
ssh $ip "[ ! -d $home ] && mkdir -p $home"
|
||||
scp -r $home/*local_table_to_2401.sh $ip:$home/
|
||||
ssh $ip "cd $home && chmod +x ./*.sh"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据，TSG24.01日志重组后数据迁移
|
||||
|
||||
# 迁移表 参数，可选值：session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00"
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00"
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数, 例如:240
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"session_record")
|
||||
# 迁移session_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"security_event")
|
||||
# 迁移security_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"monitor_event")
|
||||
# 迁移monitor_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"transaction_record")
|
||||
# 迁移transaction_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"voip_record")
|
||||
# 迁移voip_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"proxy_event")
|
||||
# 迁移proxy_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "slice_interval_minute参数必须是正确的分钟数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
# 遍历每个节点执行迁移
|
||||
for ip in `cat iplist.txt`
|
||||
do
|
||||
# 后台执行，输出日志
|
||||
echo "$ip 节点开始执行迁移"
|
||||
ssh $ip "cd $home && chmod +x ./*.sh && ./start_migrate_local_table_to_2401.sh $table '$data_start_time' '$data_end_time' $slice_interval_minute"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
table=$1
|
||||
if [ -z "$table" ]; then
|
||||
echo "缺少table参数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ips=($(cat iplist.txt))
|
||||
ips_size=${#ips[*]}
|
||||
ip_starts=$( seq 0 $(($ips_size - 1)) )
|
||||
ip_ends=$( seq 0 $(($ips_size - 1)) )
|
||||
|
||||
for ((i=0;i<$ips_size;i++))
|
||||
do
|
||||
ip_starts[$i]=0
|
||||
ip_ends[$i]=0
|
||||
done
|
||||
|
||||
while true ; do
|
||||
# 遍历每个节点
|
||||
for ((i=0;i<$ips_size;i++)); do
|
||||
ip=${ips[$i]}
|
||||
start=${ip_starts[$i]}
|
||||
end=${ip_ends[$i]}
|
||||
if [ $start -eq 0 ]; then
|
||||
info=$(ssh $ip "cat $home/log_$table.txt | grep migrate_table_start")
|
||||
if [ -n "$info" ]; then
|
||||
echo "${ip}迁移开始:${info}"
|
||||
ip_starts[$i]=1
|
||||
start=1
|
||||
fi
|
||||
fi
|
||||
if [ $start -eq 1 ] && [ $end -eq 0 ] ; then
|
||||
info=$(ssh $ip "cat $home/log_$table.txt | grep migrate_table_end")
|
||||
if [ -n "$info" ]; then
|
||||
echo "${ip}迁移结束:${info}"
|
||||
ip_ends[$i]=1
|
||||
end=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
#全部结束
|
||||
finish_cnt=0
|
||||
for ((i=0;i<$ips_size;i++)); do
|
||||
start=${ip_starts[$i]}
|
||||
end=${ip_ends[$i]}
|
||||
if [ $start -eq 1 ] && [ $end -eq 1 ] ; then
|
||||
finish_cnt=$(($finish_cnt+1))
|
||||
fi
|
||||
done
|
||||
if [ $finish_cnt -ge $ips_size ]; then
|
||||
echo "所有节点迁移结束"
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
@@ -0,0 +1 @@
|
||||
192.168.41.30
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据，TSG24.01日志重组后数据迁移
|
||||
|
||||
# 迁移表 参数，可选值：session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00"
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00"
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数, 例如:240
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"session_record")
|
||||
# 迁移session_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"security_event")
|
||||
# 迁移security_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"monitor_event")
|
||||
# 迁移monitor_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"transaction_record")
|
||||
# 迁移transaction_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"voip_record")
|
||||
# 迁移voip_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"proxy_event")
|
||||
# 迁移proxy_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "slice_interval_minute参数必须是正确的分钟数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 后台执行，输出日志
|
||||
nohup ./migrate_local_table_to_2401.sh "$table" "$data_start_time" "$data_end_time" $slice_interval_minute > "log_$table.txt" 2>&1 &
|
||||
echo "已启动迁移${table}表任务，时间范围[$data_start_time, $data_end_time], 每批迁移段分钟:$slice_interval_minute, 日志输出到:log_$table.txt。请查看日志文件确认每段数据迁移情况"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
|
||||
drop table IF EXISTS `system`.query_log_cluster on cluster ck_query;
|
||||
drop table IF EXISTS `system`.disks_cluster on cluster ck_query;
|
||||
drop table IF EXISTS `system`.columns_cluster on cluster ck_query;
|
||||
drop table IF EXISTS `system`.parts_cluster on cluster ck_query;
|
||||
drop table IF EXISTS `system`.processes_cluster on cluster ck_query;
|
||||
drop table IF EXISTS `system`.tables_cluster on cluster ck_query;
|
||||
|
||||
|
||||
create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand());
|
||||
create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand());
|
||||
create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand());
|
||||
create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand());
|
||||
CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand());
|
||||
CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand());
|
||||
alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY;
|
||||
alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY;
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,45 @@
|
||||
# -*- coding: utf-8 -
|
||||
import os
|
||||
|
||||
newSqlFilePath = "Clickhouse_TSG_建表语句_new.sql" #新表文件路径
|
||||
oldSqlFilePath = "Clickhouse_TSG_建表语句_old.sql" #旧表文件路径
|
||||
|
||||
shellCommand1 = "diff "+newSqlFilePath +" "+oldSqlFilePath#shell命令
|
||||
shellCommand2 = "diff "+oldSqlFilePath +" "+newSqlFilePath#shell命令，左右表位置互换，可能有不同的对比结果
|
||||
shellCommand = [shellCommand1,shellCommand2]
|
||||
|
||||
for s in range(0,len(shellCommand)):
|
||||
print shellCommand[s]+":"
|
||||
re = os.popen(shellCommand[s]).readlines()
|
||||
result = []
|
||||
isRight= False #未出现右箭头
|
||||
keyWord = ["PRIMARY"] #不能要的关键字
|
||||
isKeyWord = False #表示当前列表未出现关键字
|
||||
for i in range(0, len(re)): # 由于原始结果需要转换编码，所以循环转为utf8编码并且去除\n换行
|
||||
res = re[i].strip('\n')
|
||||
for j in range(0,len(keyWord)):
|
||||
if str(res).find(keyWord[j]) != -1:
|
||||
isKeyWord = True # 表示res出现关键字
|
||||
if isRight == False and str(res).find(">", 0, 1) == 0:
|
||||
isRight = True
|
||||
result.append(res)
|
||||
elif isRight == True and str(res).find(">",0,1) == -1 :
|
||||
if isKeyWord ==False:
|
||||
print result
|
||||
result = []
|
||||
result.append(res)
|
||||
isRight = False
|
||||
isKeyWord = False
|
||||
else:
|
||||
result.append(res)
|
||||
if i == len(re) -1 :
|
||||
if isKeyWord == False:
|
||||
print result
|
||||
print ""
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
8
tsg_olap/installation/clickhouse/system.sql
Normal file
8
tsg_olap/installation/clickhouse/system.sql
Normal file
@@ -0,0 +1,8 @@
|
||||
create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand());
|
||||
create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand());
|
||||
create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand());
|
||||
create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand());
|
||||
CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand());
|
||||
CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand());
|
||||
alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY;
|
||||
alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY;
|
||||
4338
tsg_olap/installation/clickhouse/tsg_olap_clickhouse_ddl.sql
Normal file
4338
tsg_olap/installation/clickhouse/tsg_olap_clickhouse_ddl.sql
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,21 @@
|
||||
SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
|
||||
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
|
||||
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
|
||||
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
|
||||
FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT log_id, recv_time, vsys_id, timestamp_us, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
|
||||
FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
38
tsg_olap/installation/clickhouse/性能测试脚本/ck_query.sh
Normal file
38
tsg_olap/installation/clickhouse/性能测试脚本/ck_query.sh
Normal file
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
|
||||
sql_file=$1 # sql文件路径
|
||||
operation=$2 # 1是批量查询，2是单条依次查询
|
||||
startTime="'2024-01-28 15:11:32'"
|
||||
endTime="'2024-01-29 15:11:32'"
|
||||
query_batch=1 # 批量查询同时查询的数量
|
||||
host=192.168.44.12
|
||||
port=9001
|
||||
user=default
|
||||
password=galaxy2019
|
||||
query_count=$(wc -l < "$sql_file")
|
||||
|
||||
if [ $operation -eq 1 ];then
|
||||
start_time=$(date +%s.%N)
|
||||
cat $sql_file | sed "s/start_time/$startTime/g" | sed "s/end_time/$endTime/g" | awk -F ';' '{print $2}' | clickhouse-benchmark -h $host -p $port --user=$user --password=$password -i $query_count -d 0 -c $query_batch
|
||||
end_time=$(date +%s.%N)
|
||||
total_time=$(echo $(printf "%.3f" "$(echo "scale=3; $end_time - $start_time" | bc)"))
|
||||
echo "Elapsed Time: $total_time seconds"
|
||||
echo "Avg Elapsed Time: $(echo $(printf "%.3f" "$(echo "scale=3; $total_time / $query_count" | bc)")) seconds"
|
||||
exit 0
|
||||
else
|
||||
total_time=0
|
||||
set -f
|
||||
while IFS= read -r line
|
||||
do
|
||||
query=$(echo $line | sed "s/start_time/$startTime/g" | sed "s/end_time/$endTime/g")
|
||||
query_name=$(echo $query | awk -F ';' '{print $1}')
|
||||
sql=$(echo $query | awk -F ';' '{print $2}')
|
||||
query_time=$(clickhouse-client --host=$host --port=$port --user=$user --password=$password -t --query="$sql" 2>&1 >/dev/null)
|
||||
echo "Query: $query_name"
|
||||
echo "Elapsed Time: $query_time seconds"
|
||||
total_time=$(echo "$total_time + $query_time" | bc)
|
||||
done < $sql_file
|
||||
echo "Total Elapsed Time: $total_time seconds"
|
||||
echo "Avg Elapsed Time: $(echo $(printf "%.3f" "$(echo "scale=3; $total_time / $query_count" | bc)")) seconds"
|
||||
exit 0
|
||||
fi
|
||||
66
tsg_olap/installation/clickhouse/性能测试脚本/query-new.sql
Normal file
66
tsg_olap/installation/clickhouse/性能测试脚本/query-new.sql
Normal file
@@ -0,0 +1,66 @@
|
||||
Q01.All Fields sub Query (default) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) LIMIT 30;
|
||||
Q02.All Fields sub Query order by Time desc ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q03.All Fields sub Query order by Time asc ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) ORDER BY recv_time ASC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) ORDER BY recv_time ASC LIMIT 30 ;
|
||||
Q04.All Fields sub Query by Filter(log_id=434228307888582660) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND log_id = 434228307888582660 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND log_id = 434228307888582660 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q05.All Fields sub Query by Filter(client_port=52607) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND client_port = 52607 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND client_port = 52607 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q06.All Fields sub Query by Filter(server_port=443) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_port = 443 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_port = 443 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q07.All Fields sub Query by Filter(sent_pkts>5) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND sent_pkts > 5 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND sent_pkts > 5 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q08.All Fields sub Query by Filter(received_pkts>5) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND received_pkts > 5 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND received_pkts > 5 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q09.All Fields sub Query by Filter(sent_bytes>100) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND sent_bytes > 100 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND sent_bytes > 100 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q10.All Fields sub Query by Filter(received_bytes<200) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND received_bytes < 200 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND received_bytes < 200 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q11.All Fields sub Query by Filter(decoded_as='DNS');SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND decoded_as = 'DNS' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND decoded_as = 'DNS' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q12.All Fields sub Query by Filter(tcp_handshake_latency_ms>200) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND tcp_handshake_latency_ms > 200 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND tcp_handshake_latency_ms > 200 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q13.All Fields sub Query by Filter(duration_ms>10000);SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND duration_ms > 10000 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND duration_ms > 10000 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q14.All Fields sub Query by Filter(session_id=434228307888582660);SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND session_id = 434228307888582660 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND session_id = 434228307888582660 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q15.All Fields sub Query by Filter(tcp_client_isn=2857077935);SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND tcp_client_isn = 2857077935 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND tcp_client_isn = 2857077935 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q16.All Fields sub Query by Filter(tcp_server_isn=0);SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND tcp_server_isn = 0 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND tcp_server_isn = 0 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q17.All Fields sub Query by Filter(mail_account='abc@xx.com');SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND mail_account = 'abc@xx.com' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND mail_account = 'abc@xx.com' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q18.All Fields sub Query by Filter(mail_subject='test') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND mail_subject = 'test' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND mail_subject = 'test' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q19.All Fields sub Query by Filter(server_domain='qq.com') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_domain = 'qq.com' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_domain = 'qq.com' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q20.All Fields sub Query by Filter(dns_qname='qbwup.imtt.qq.com');SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND dns_qname = 'qbwup.imtt.qq.com' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND dns_qname = 'qbwup.imtt.qq.com' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q21.All Fields sub Query by Filter(ssl_sni='note.youdao.com');SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND ssl_sni = 'note.youdao.com' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND ssl_sni = 'note.youdao.com' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q22.All Fields sub Query by Filter(ssl_handshake_latency_ms>100) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND ssl_handshake_latency_ms > 100 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND ssl_handshake_latency_ms > 100 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q23.All Fields sub Query by Filter(ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND ssl_ja3_hash = 'a0e9f5d64349fb13191bc781f81f42e1' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND ssl_ja3_hash = 'a0e9f5d64349fb13191bc781f81f42e1' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q24.All Fields sub Query by Filter(server_ip='111.10.53.14' and server_port=443) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_ip = '111.10.53.14' AND server_port = 443 ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_ip = '111.10.53.14' AND server_port = 443 ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q25.All Fields sub Query by Filter(mail_account like 'abc@%');SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND mail_account LIKE 'abc@%' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND mail_account LIKE 'abc@%' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q26.All Fields sub Query by Filter(ssl_sni like '%youdao.com');SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND ssl_sni LIKE '%youdao.com' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND ssl_sni LIKE '%youdao.com' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q27.All Fields sub Query by Filter(server_domain like '%baidu.com%') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_domain LIKE '%baidu.com%' ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_domain LIKE '%baidu.com%' ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q28.All Fields sub Query by Filter(server_port not in (80,443)) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time IN (SELECT recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_port NOT IN (80, 443) ORDER BY recv_time DESC LIMIT 30) AND recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_port NOT IN (80, 443) ORDER BY recv_time DESC LIMIT 30 ;
|
||||
Q29.ip_protocol top10 ;select ip_protocol ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by ip_protocol order by count() desc limit 10 ;
|
||||
Q30.decoded_as top10 ;select decoded_as ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by decoded_as order by count() desc limit 10 ;
|
||||
Q31.client_ip top10 ;select client_ip ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by client_ip order by count() desc limit 10 ;
|
||||
Q32.client_port top10 ;select client_port ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by client_port order by count() desc limit 10 ;
|
||||
Q33.subscriber_id top10 ;select subscriber_id ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by subscriber_id order by count() desc limit 10 ;
|
||||
Q34.server_ip top10 ;select server_ip ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by server_ip order by count() desc limit 10 ;
|
||||
Q35.server_port top10 ;select server_port ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by server_port order by count() desc limit 10 ;
|
||||
Q36.app top10 ;select app ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by app order by count() desc limit 10 ;
|
||||
Q37.sent_pkts top10 ;select sent_pkts ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by sent_pkts order by count() desc limit 10 ;
|
||||
Q38.received_pkts top10 ;select received_pkts ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by received_pkts order by count() desc limit 10 ;
|
||||
Q39.http_url top10 ;select http_url ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by http_url order by count() desc limit 10 ;
|
||||
Q40.http_host top10 ;select http_host ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by http_host order by count() desc limit 10 ;
|
||||
Q41.server_domain top10 ;select server_domain ,count() from tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by server_domain order by count() desc limit 10 ;
|
||||
Q42.ip_protocol top10 by Filter(client_ip='36.189.226.21'); select ip_protocol ,count() from tsg_galaxy_v3.session_record where client_ip='36.189.226.21' and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by ip_protocol order by count() desc limit 10 ;
|
||||
Q43.client_ip top10 by Filter(server_ip='8.8.8.8');select client_ip ,count() from tsg_galaxy_v3.session_record where server_ip='8.8.8.8' and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by client_ip order by count() desc limit 10 ;
|
||||
Q44.client_port top10 by Filter(server_port=443);select client_port ,count() from tsg_galaxy_v3.session_record where server_port=443 and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by client_port order by count() desc limit 10 ;
|
||||
Q45.subscriber_id top10 by Filter(sent_bytes > 100);select subscriber_id ,count() from tsg_galaxy_v3.session_record where sent_bytes > 100 and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by subscriber_id order by count() desc limit 10 ;
|
||||
Q46.server_port top10 by Filter(received_bytes<200);select server_port ,count() from tsg_galaxy_v3.session_record where received_bytes<200 and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by server_port order by count() desc limit 10 ;
|
||||
Q47.app top10 by Filter(server_domain like '%baidu.com%');select app ,count() from tsg_galaxy_v3.session_record where server_domain like '%baidu.com%' and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) and server_domain like '%baidu.com%' group by app order by count() desc limit 10 ;
|
||||
Q48.sent_pkts top10 by Filter(ssl_sni='note.youdao.com');select sent_pkts ,count() from tsg_galaxy_v3.session_record where ssl_sni='note.youdao.com' and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by sent_pkts order by count() desc limit 10 ;
|
||||
Q49.received_pkts top10 by Filter(server_ip='111.10.53.14' and server_port=443);select received_pkts ,count() from tsg_galaxy_v3.session_record where server_ip='111.10.53.14' and server_port=443 and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by received_pkts order by count() desc limit 10 ;
|
||||
Q50.http_url top10 by Filter(sent_bytes>100);select http_url ,count() from tsg_galaxy_v3.session_record where sent_bytes>100 and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by http_url order by count() desc limit 10 ;
|
||||
Q51.http_host top10 by Filter(server_ip='8.8.8.8');select http_host ,count() from tsg_galaxy_v3.session_record where server_ip='8.8.8.8' and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by http_host order by count() desc limit 10 ;
|
||||
Q52.server_domain top10 by Filter(decoded_as='HTTP');select server_domain ,count() from tsg_galaxy_v3.session_record where decoded_as='HTTP' and recv_time >= toUnixTimestamp(start_time) and recv_time < toUnixTimestamp(end_time) group by server_domain order by count() desc limit 10 ;
|
||||
Q53.Bandwidth Trend (Time Grain 60 second) nofilter ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))) AS stat_time, decoded_as AS type, sum(sent_bytes + received_bytes) AS bytes, sum(sent_pkts + received_pkts) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))), decoded_as LIMIT 100000 ;
|
||||
Q54.Bandwidth Trend (Time Grain 60 second) by Filter(client_ip='36.189.226.21') ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))) AS stat_time, decoded_as AS type, sum(sent_bytes + received_bytes) AS bytes, sum(sent_pkts + received_pkts) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND client_ip='36.189.226.21' GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))), decoded_as LIMIT 100000 ;
|
||||
Q55.Bandwidth Trend (Time Grain 60 second) by Filter(server_ip='8.8.8.8') ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))) AS stat_time, decoded_as AS type, sum(sent_bytes + received_bytes) AS bytes, sum(sent_pkts + received_pkts) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_ip='8.8.8.8' GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))), decoded_as LIMIT 100000 ;
|
||||
Q56.Bandwidth Trend (Time Grain 60 second) by Filter(server_domain='microsoft.com') ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))) AS stat_time, decoded_as AS type, sum(sent_bytes + received_bytes) AS bytes, sum(sent_pkts + received_pkts) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_domain='microsoft.com' GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))), decoded_as LIMIT 100000 ;
|
||||
Q57.Bandwidth Trend (Time Grain 60 second) by Filter(server_ip='111.10.53.14' and server_port=443);SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))) AS stat_time, decoded_as AS type, sum(sent_bytes + received_bytes) AS bytes, sum(sent_pkts + received_pkts) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_ip='111.10.53.14' and server_port=443 GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(recv_time)), INTERVAL 60 SECOND))))), decoded_as LIMIT 100000 ;
|
||||
Q58.Metrics Query by Filter(decoded_as='HTTP') ;SELECT ROUND(AVG(http_response_latency_ms)) AS http_response_latency FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND decoded_as = 'HTTP' ;
|
||||
Q59.Metrics Query by Filter(app = 'qq_r2');SELECT ROUND(SUM(tcp_c2s_rtx_pkts + tcp_s2c_rtx_pkts)/SUM(sent_pkts + received_pkts),4) AS avg_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND app = 'qq_r2' ;
|
||||
Q60.Metrics Query by Filter(server_domain='qq.com') ;SELECT ROUND(SUM(tcp_c2s_rtx_pkts + tcp_s2c_rtx_pkts)/SUM(sent_pkts + received_pkts),4) AS avg_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_domain = 'qq.com' ;
|
||||
Q61.Metrics Query by Filter(client_ip='116.178.223.59');SELECT ROUND(SUM(tcp_c2s_rtx_pkts + tcp_s2c_rtx_pkts)/SUM(sent_pkts + received_pkts),4) AS avg_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND client_ip = '116.178.223.59' ;
|
||||
Q62.Metrics Query by Filter(server_ip='8.8.8.8');SELECT ROUND(SUM(tcp_c2s_rtx_pkts + tcp_s2c_rtx_pkts)/SUM(sent_pkts + received_pkts),4) AS avg_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_ip = '8.8.8.8' ;
|
||||
Q63.Metrics Query by Filter(app = 'qq_r2') group by recv_time;SELECT ROUND(SUM(tcp_c2s_rtx_pkts + tcp_s2c_rtx_pkts)/SUM(sent_pkts + received_pkts),4) as max_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND app = 'qq_r2' GROUP by recv_time order BY max_pkt_retrans_percent desc limit 1 ;
|
||||
Q64.Metrics Query by Filter(server_domain='qq.com') group by recv_time;SELECT ROUND(SUM(tcp_c2s_rtx_pkts + tcp_s2c_rtx_pkts)/SUM(sent_pkts + received_pkts),4) as max_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_domain = 'qq.com' GROUP by recv_time order BY max_pkt_retrans_percent desc limit 1 ;
|
||||
Q65.Metrics Query by Filter(client_ip='116.178.223.59') group by recv_time;SELECT ROUND(SUM(tcp_c2s_rtx_pkts + tcp_s2c_rtx_pkts)/SUM(sent_pkts + received_pkts),4) as max_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND client_ip = '116.178.223.59' GROUP by recv_time order BY max_pkt_retrans_percent desc limit 1 ;
|
||||
Q66.Metrics Query by Filter(server_ip='8.8.8.8') group by recv_time;SELECT ROUND(SUM(tcp_c2s_rtx_pkts + tcp_s2c_rtx_pkts)/SUM(sent_pkts + received_pkts),4) as max_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE recv_time >= toUnixTimestamp(start_time) AND recv_time < toUnixTimestamp(end_time) AND server_ip = '8.8.8.8' GROUP by recv_time order BY max_pkt_retrans_percent desc limit 1 ;
|
||||
74
tsg_olap/installation/clickhouse/性能测试脚本/query-old.sql
Normal file
74
tsg_olap/installation/clickhouse/性能测试脚本/query-old.sql
Normal file
@@ -0,0 +1,74 @@
|
||||
Q01.All Fields sub Query (default) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) LIMIT 30 ;
|
||||
Q02.All Fields sub Query order by Time desc ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q03.All Fields sub Query order by Time asc ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time ASC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time ASC LIMIT 30 ;
|
||||
Q04.All Fields sub Query by Filter(common_log_id=434228307888582660) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_log_id = 434228307888582660 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_log_id = 434228307888582660 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q05.All Fields sub Query by Filter(common_internal_ip='223.116.37.192') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_internal_ip = '223.116.37.192' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_internal_ip = '223.116.37.192' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q06.All Fields sub Query by Filter(common_external_ip='111.10.53.14') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_external_ip = '111.10.53.14' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_external_ip = '111.10.53.14' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q07.All Fields sub Query by Filter(common_client_port=52607) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_client_port = 52607 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_client_port = 52607 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q08.All Fields sub Query by Filter(common_server_port=443) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_port = 443 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_port = 443 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q09.All Fields sub Query by Filter(common_c2s_pkt_num>5) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_c2s_pkt_num > 5 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_c2s_pkt_num > 5 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q10.All Fields sub Query by Filter(common_s2c_pkt_num>5) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_s2c_pkt_num > 5 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_s2c_pkt_num > 5 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q11.All Fields sub Query by Filter(common_c2s_byte_num>100) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_c2s_byte_num > 100 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_c2s_byte_num > 100 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q12.All Fields sub Query by Filter(common_s2c_byte_num<200) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_s2c_byte_num < 200 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_s2c_byte_num < 200 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q13.All Fields sub Query by Filter(common_schema_type='DNS') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_schema_type = 'DNS' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_schema_type = 'DNS' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q14.All Fields sub Query by Filter(common_establish_latency_ms>200) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_establish_latency_ms > 200 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_establish_latency_ms > 200 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q15.All Fields sub Query by Filter(common_con_duration_ms>10000) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_con_duration_ms > 10000 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_con_duration_ms > 10000 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q16.All Fields sub Query by Filter(common_stream_trace_id=434228307888582660) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_stream_trace_id = 434228307888582660 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_stream_trace_id = 434228307888582660 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q17.All Fields sub Query by Filter(common_tcp_client_isn=2857077935) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_tcp_client_isn = 2857077935 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_tcp_client_isn = 2857077935 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q18.All Fields sub Query by Filter(common_tcp_server_isn=0) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_tcp_server_isn = 0 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_tcp_server_isn = 0 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q19.All Fields sub Query by Filter(mail_account='abc@xx.com') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND mail_account = 'abc@xx.com' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND mail_account = 'abc@xx.com' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q20.All Fields sub Query by Filter(mail_subject='test') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND mail_subject = 'test' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND mail_subject = 'test' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q21.All Fields sub Query by Filter(http_domain='qq.com') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND http_domain = 'qq.com' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND http_domain = 'qq.com' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q22.All Fields sub Query by Filter(dns_qname='qbwup.imtt.qq.com') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND dns_qname = 'qbwup.imtt.qq.com' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND dns_qname = 'qbwup.imtt.qq.com' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q23.All Fields sub Query by Filter(ssl_sni='note.youdao.com') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND ssl_sni = 'note.youdao.com' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND ssl_sni = 'note.youdao.com' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q24.All Fields sub Query by Filter(ssl_con_latency_ms>100) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND ssl_con_latency_ms > 100 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND ssl_con_latency_ms > 100 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q25.All Fields sub Query by Filter(ssl_ja3_hash='9b02ebd3a43b62d825e1ac605b621dc8') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND ssl_ja3_hash = '9b02ebd3a43b62d825e1ac605b621dc8' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND ssl_ja3_hash = '9b02ebd3a43b62d825e1ac605b621dc8' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q26.All Fields sub Query by Filter(common_server_ip='111.10.53.14' and common_server_port=443) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_ip = '111.10.53.14' AND common_server_port = 443 ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_ip = '111.10.53.14' AND common_server_port = 443 ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q27.All Fields sub Query by Filter(mail_account like 'abc@%') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND mail_account LIKE 'abc@%' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND mail_account LIKE 'abc@%' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q28.All Fields sub Query by Filter(ssl_sni like '%youdao.com') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND ssl_sni LIKE '%youdao.com' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND ssl_sni LIKE '%youdao.com' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q29.All Fields sub Query by Filter(http_domain like '%baidu.com%') ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND http_domain LIKE '%baidu.com%' ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND http_domain LIKE '%baidu.com%' ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q30.All Fields sub Query by Filter(common_server_port not in (80,443)) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_port NOT IN (80, 443) ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_port NOT IN (80, 443) ORDER BY common_recv_time DESC LIMIT 30 ;
|
||||
Q31.All Fields sub Query (sub query by time) ;SELECT * FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE toDateTime(common_recv_time) IN (SELECT toDateTime(common_recv_time) FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time DESC LIMIT 30) AND toDateTime(common_recv_time) IN (SELECT toDateTime(common_recv_time) FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time IN (SELECT common_recv_time FROM tsg_galaxy_v3.session_record AS session_record WHERE toDateTime(common_recv_time) IN (SELECT toDateTime(common_recv_time) FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time DESC LIMIT 30) AND common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) ORDER BY common_recv_time DESC LIMIT 30;
|
||||
Q32.common_l4_protocol top10 ;select common_l4_protocol ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_l4_protocol order by count() desc limit 10 ;
|
||||
Q33.common_schema_type top10 ;select common_schema_type ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_schema_type order by count() desc limit 10 ;
|
||||
Q34.common_client_ip top10 ;select common_client_ip ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_client_ip order by count() desc limit 10 ;
|
||||
Q35.common_client_port top10 ;select common_client_port ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_client_port order by count() desc limit 10 ;
|
||||
Q36.common_subscriber_id top10 ;select common_subscriber_id ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_subscriber_id order by count() desc limit 10 ;
|
||||
Q37.common_server_ip top10 ;select common_server_ip ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_server_ip order by count() desc limit 10 ;
|
||||
Q38.common_server_port top10 ;select common_server_port ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_server_port order by count() desc limit 10 ;
|
||||
Q39.common_app_id top10 ;select common_app_id ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_app_id order by count() desc limit 10 ;
|
||||
Q40.common_app_label top10 ;select common_app_label ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_app_label order by count() desc limit 10 ;
|
||||
Q41.common_l7_protocol top10 ;select common_l7_protocol ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_l7_protocol order by count() desc limit 10 ;
|
||||
Q42.common_c2s_pkt_num top10 ;select common_c2s_pkt_num ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_c2s_pkt_num order by count() desc limit 10 ;
|
||||
Q43.common_s2c_pkt_num top10 ;select common_s2c_pkt_num ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_s2c_pkt_num order by count() desc limit 10 ;
|
||||
Q44.http_url top10 ;select http_url ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by http_url order by count() desc limit 10 ;
|
||||
Q45.http_host top10 ;select http_host ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by http_host order by count() desc limit 10 ;
|
||||
Q46.http_domain top10 ;select http_domain ,count() from tsg_galaxy_v3.session_record where common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by http_domain order by count() desc limit 10 ;
|
||||
Q47.common_l4_protocol top10 by Filter(common_client_ip='36.189.226.21') ;select common_l4_protocol ,count() from tsg_galaxy_v3.session_record where common_client_ip='36.189.226.21' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_l4_protocol order by count() desc limit 10 ;
|
||||
Q48.common_schema_type top10 by Filter(common_internal_ip='223.116.37.192') ;select common_schema_type ,count() from tsg_galaxy_v3.session_record where common_internal_ip='223.116.37.192' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_schema_type order by count() desc limit 10 ;
|
||||
Q49.common_client_ip top10 by Filter(common_server_ip='8.8.8.8') ;select common_client_ip ,count() from tsg_galaxy_v3.session_record where common_server_ip='8.8.8.8' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_client_ip order by count() desc limit 10 ;
|
||||
Q50.common_client_port top10 by Filter(common_server_port=443) ;select common_client_port ,count() from tsg_galaxy_v3.session_record where common_server_port=443 and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_client_port order by count() desc limit 10 ;
|
||||
Q51.common_subscriber_id top10 by Filter(common_c2s_byte_num > 100) ;select common_subscriber_id ,count() from tsg_galaxy_v3.session_record where common_c2s_byte_num > 100 and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_subscriber_id order by count() desc limit 10 ;
|
||||
Q52.common_server_ip top10 by Filter(common_external_ip='111.10.53.14') ;select common_server_ip ,count() from tsg_galaxy_v3.session_record where common_external_ip='111.10.53.14' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_server_ip order by count() desc limit 10 ;
|
||||
Q53.common_server_port top10 by Filter(common_s2c_byte_num<200) ;select common_server_port ,count() from tsg_galaxy_v3.session_record where common_s2c_byte_num<200 and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_server_port order by count() desc limit 10 ;
|
||||
Q54.common_app_id top10 by Filter(http_domain like '%baidu.com%') ;select common_app_id ,count() from tsg_galaxy_v3.session_record where http_domain like '%baidu.com%' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) and http_domain like '%baidu.com%' group by common_app_id order by count() desc limit 10 ;
|
||||
Q55.common_app_label top10 by Filter(common_server_port not in (80,443)) ;select common_app_label ,count() from tsg_galaxy_v3.session_record where common_server_port not in (80,443) and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_app_label order by count() desc limit 10 ;
|
||||
Q56.common_l7_protocol top10 by Filter(http_domain not like '%microsoft.com') ;select common_l7_protocol ,count() from tsg_galaxy_v3.session_record where http_domain not like '%microsoft.com' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_l7_protocol order by count() desc limit 10 ;
|
||||
Q57.common_c2s_pkt_num top10 by Filter(ssl_sni='note.youdao.com') ;select common_c2s_pkt_num ,count() from tsg_galaxy_v3.session_record where ssl_sni='note.youdao.com' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_c2s_pkt_num order by count() desc limit 10 ;
|
||||
Q58.common_s2c_pkt_num top10 by Filter(common_server_ip='111.10.53.14' and common_server_port=443) ;select common_s2c_pkt_num ,count() from tsg_galaxy_v3.session_record where common_server_ip='111.10.53.14' and common_server_port=443 and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by common_s2c_pkt_num order by count() desc limit 10 ;
|
||||
Q59.http_url top10 by Filter(common_c2s_byte_num>100) ;select http_url ,count() from tsg_galaxy_v3.session_record where common_c2s_byte_num>100 and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by http_url order by count() desc limit 10 ;
|
||||
Q60.http_host top10 by Filter(common_server_ip='8.8.8.8') ;select http_host ,count() from tsg_galaxy_v3.session_record where common_server_ip='8.8.8.8' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by http_host order by count() desc limit 10 ;
|
||||
Q61.http_domain top10 by Filter(common_schema_type='HTTP') ;select http_domain ,count() from tsg_galaxy_v3.session_record where common_schema_type='HTTP' and common_recv_time >= toUnixTimestamp(start_time) and common_recv_time < toUnixTimestamp(end_time) group by http_domain order by count() desc limit 10 ;
|
||||
Q62.Bandwidth Trend (Time Grain 60 second) nofilter ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))), common_schema_type LIMIT 100000 ;
|
||||
Q63.Bandwidth Trend (Time Grain 60 second) by Filter(common_client_ip='36.189.226.21') ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_client_ip='36.189.226.21' GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))), common_schema_type LIMIT 100000 ;
|
||||
Q64.Bandwidth Trend (Time Grain 60 second) by Filter(common_server_ip='8.8.8.8') ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_ip='8.8.8.8' GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))), common_schema_type LIMIT 100000 ;
|
||||
Q65.Bandwidth Trend (Time Grain 60 second) by Filter(http_domain='microsoft.com') ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND http_domain='microsoft.com' GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))), common_schema_type LIMIT 100000 ;
|
||||
Q66.Bandwidth Trend (Time Grain 60 second) by Filter(common_server_ip='111.10.53.14' and common_server_port=443) ;SELECT toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM tsg_galaxy_v3.session_record AS session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_ip='111.10.53.14' and common_server_port=443 GROUP BY toDateTime(toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(toUnixTimestamp(common_recv_time)), INTERVAL 60 SECOND))))), common_schema_type LIMIT 100000 ;
|
||||
Q67.Metrics Query by Filter(appid='2815') ;SELECT ROUND(SUM(common_c2s_pkt_retrans + common_s2c_pkt_retrans)/SUM(common_c2s_pkt_num + common_s2c_pkt_num),4) AS avg_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_app_id = '2815' ;
|
||||
Q68.Metrics Query by Filter(http_domain='qq.com') ;SELECT ROUND(SUM(common_c2s_pkt_retrans + common_s2c_pkt_retrans)/SUM(common_c2s_pkt_num + common_s2c_pkt_num),4) AS avg_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND http_domain = 'qq.com' ;
|
||||
Q69.Metrics Query by Filter(common_client_ip='116.178.223.59') ;SELECT ROUND(SUM(common_c2s_pkt_retrans + common_s2c_pkt_retrans)/SUM(common_c2s_pkt_num + common_s2c_pkt_num),4) AS avg_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_client_ip = '116.178.223.59' ;
|
||||
Q70.Metrics Query by Filter(common_server_ip='8.8.8.8') ;SELECT ROUND(SUM(common_c2s_pkt_retrans + common_s2c_pkt_retrans)/SUM(common_c2s_pkt_num + common_s2c_pkt_num),4) AS avg_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_ip = '8.8.8.8' ;
|
||||
Q71.Metrics Query by Filter(appid='2815') group by common_recv_time ;SELECT ROUND(SUM(common_c2s_pkt_retrans + common_s2c_pkt_retrans)/SUM(common_c2s_pkt_num + common_s2c_pkt_num),4) as max_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_app_id = '2815' GROUP by common_recv_time order BY max_pkt_retrans_percent desc limit 1 ;
|
||||
Q72.Metrics Query by Filter(http_domain='qq.com') group by common_recv_time ;SELECT ROUND(SUM(common_c2s_pkt_retrans + common_s2c_pkt_retrans)/SUM(common_c2s_pkt_num + common_s2c_pkt_num),4) as max_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND http_domain = 'qq.com' GROUP by common_recv_time order BY max_pkt_retrans_percent desc limit 1 ;
|
||||
Q73.Metrics Query by Filter(common_client_ip='116.178.223.59') group by common_recv_time ;SELECT ROUND(SUM(common_c2s_pkt_retrans + common_s2c_pkt_retrans)/SUM(common_c2s_pkt_num + common_s2c_pkt_num),4) as max_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_client_ip = '116.178.223.59' GROUP by common_recv_time order BY max_pkt_retrans_percent desc limit 1 ;
|
||||
Q74.Metrics Query by Filter(common_server_ip='8.8.8.8') group by common_recv_time ;SELECT ROUND(SUM(common_c2s_pkt_retrans + common_s2c_pkt_retrans)/SUM(common_c2s_pkt_num + common_s2c_pkt_num),4) as max_pkt_retrans_percent FROM tsg_galaxy_v3.session_record WHERE common_recv_time >= toUnixTimestamp(start_time) AND common_recv_time < toUnixTimestamp(end_time) AND common_server_ip = '8.8.8.8' GROUP by common_recv_time order BY max_pkt_retrans_percent desc limit 1;
|
||||
1
tsg_olap/installation/clickhouse/数据迁移脚本/iplist.txt
Normal file
1
tsg_olap/installation/clickhouse/数据迁移脚本/iplist.txt
Normal file
@@ -0,0 +1 @@
|
||||
192.168.41.30
|
||||
339
tsg_olap/installation/clickhouse/数据迁移脚本/mig.sh
Normal file
339
tsg_olap/installation/clickhouse/数据迁移脚本/mig.sh
Normal file
@@ -0,0 +1,339 @@
|
||||
#!/bin/bash
|
||||
|
||||
#本脚本安装逐台服务器逐时间段的顺序迁移clickhouse数据
|
||||
data_start_time="2021-09-10 00:00:00" #迁移数据common_recv_time 起始时间
|
||||
data_end_time="2021-09-15 23:59:59"
|
||||
query_timeout=36000 # 执行insert 的超时时间秒
|
||||
data_split_part=170 #切割次数 每段时间区间= (data_start_time-data_end_time)/data_split_part
|
||||
ck_username="default" #ck用户名
|
||||
ck_password="galaxy2019" #ck密码
|
||||
data_destination_table="tsg_galaxy_v3.session_record_local_t5" #目的表名
|
||||
data_source_table="tsg_galaxy_xj.connection_record_log_local" #源表名
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
timestamp_start=`date --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --date="$data_end_time" +%s`
|
||||
|
||||
for ip in `cat iplist.txt`
|
||||
do echo " $ip start"
|
||||
slice_time_interval=$(((timestamp_end-timestamp_start)/data_split_part))
|
||||
if [ $timestamp_start -lt $timestamp_end ];then
|
||||
for ((i = 0 ; i < $data_split_part ; i++))
|
||||
do
|
||||
input_time_start=$((timestamp_start+i*slice_time_interval))
|
||||
input_time_end=$((input_time_start+slice_time_interval))
|
||||
clickhouse-client -h $ip --port 9001 -m -u $ck_username --password $ck_password --max_final_threads=1 --max_insert_threads=1 --max_threads=1 --max_execution_time=$query_timeout --query="insert
|
||||
into
|
||||
$data_destination_table
|
||||
(common_log_id ,
|
||||
common_service ,
|
||||
common_recv_time ,
|
||||
common_direction ,
|
||||
common_l4_protocol ,
|
||||
common_address_type ,
|
||||
common_schema_type ,
|
||||
common_policy_id ,
|
||||
common_user_tags ,
|
||||
common_action ,
|
||||
common_sub_action ,
|
||||
common_user_region ,
|
||||
common_client_ip ,
|
||||
common_client_port ,
|
||||
common_internal_ip ,
|
||||
common_entrance_id ,
|
||||
common_device_id ,
|
||||
common_isp ,
|
||||
common_device_tag ,
|
||||
common_data_center ,
|
||||
common_encapsulation ,
|
||||
common_sled_ip ,
|
||||
common_client_location ,
|
||||
common_client_asn ,
|
||||
common_subscriber_id ,
|
||||
common_server_ip ,
|
||||
common_server_port ,
|
||||
common_external_ip ,
|
||||
common_server_location ,
|
||||
common_server_asn ,
|
||||
common_protocol_label ,
|
||||
common_app_label ,
|
||||
common_l7_protocol ,
|
||||
common_sessions ,
|
||||
common_c2s_pkt_num ,
|
||||
common_s2c_pkt_num ,
|
||||
common_c2s_byte_num ,
|
||||
common_s2c_byte_num ,
|
||||
common_start_time ,
|
||||
common_end_time ,
|
||||
common_establish_latency_ms ,
|
||||
common_con_duration_ms ,
|
||||
common_stream_dir ,
|
||||
common_address_list ,
|
||||
common_has_dup_traffic ,
|
||||
common_stream_error ,
|
||||
common_stream_trace_id ,
|
||||
common_link_info_c2s ,
|
||||
common_link_info_s2c ,
|
||||
common_c2s_ipfrag_num ,
|
||||
common_s2c_ipfrag_num ,
|
||||
common_c2s_tcp_lostlen ,
|
||||
common_s2c_tcp_lostlen ,
|
||||
common_c2s_tcp_unorder_num ,
|
||||
common_s2c_tcp_unorder_num ,
|
||||
common_first_ttl ,
|
||||
common_processing_time ,
|
||||
http_url ,
|
||||
http_host ,
|
||||
http_domain ,
|
||||
http_request_line ,
|
||||
http_response_line ,
|
||||
http_request_header ,
|
||||
http_response_header ,
|
||||
http_request_body ,
|
||||
http_response_body ,
|
||||
http_request_body_key ,
|
||||
http_response_body_key ,
|
||||
http_proxy_flag ,
|
||||
http_sequence ,
|
||||
http_snapshot ,
|
||||
http_cookie ,
|
||||
http_referer ,
|
||||
http_user_agent ,
|
||||
http_content_length ,
|
||||
http_content_type ,
|
||||
http_set_cookie ,
|
||||
http_version ,
|
||||
http_session_duration_ms ,
|
||||
http_action_file_size ,
|
||||
mail_protocol_type ,
|
||||
mail_account ,
|
||||
mail_to_cmd ,
|
||||
mail_from_cmd ,
|
||||
mail_from ,
|
||||
mail_to ,
|
||||
mail_cc ,
|
||||
mail_bcc ,
|
||||
mail_subject ,
|
||||
mail_subject_charset ,
|
||||
mail_content ,
|
||||
mail_content_charset ,
|
||||
mail_attachment_name ,
|
||||
mail_attachment_name_charset ,
|
||||
mail_attachment_content ,
|
||||
mail_eml_file ,
|
||||
mail_snapshot ,
|
||||
dns_message_id ,
|
||||
dns_qr ,
|
||||
dns_opcode ,
|
||||
dns_aa ,
|
||||
dns_tc ,
|
||||
dns_rd ,
|
||||
dns_ra ,
|
||||
dns_rcode ,
|
||||
dns_qdcount ,
|
||||
dns_ancount ,
|
||||
dns_nscount ,
|
||||
dns_arcount ,
|
||||
dns_qname ,
|
||||
dns_qtype ,
|
||||
dns_qclass ,
|
||||
dns_cname ,
|
||||
dns_sub ,
|
||||
dns_rr ,
|
||||
ssl_version ,
|
||||
ssl_sni ,
|
||||
ssl_san ,
|
||||
ssl_cn ,
|
||||
ssl_pinningst ,
|
||||
ssl_intercept_state ,
|
||||
ssl_server_side_latency ,
|
||||
ssl_client_side_latency ,
|
||||
ssl_server_side_version ,
|
||||
ssl_client_side_version ,
|
||||
ssl_cert_verify ,
|
||||
ssl_error ,
|
||||
ssl_con_latency_ms ,
|
||||
ssl_ja3_fingerprint ,
|
||||
ssl_ja3_hash ,
|
||||
quic_version ,
|
||||
quic_sni ,
|
||||
quic_user_agent ,
|
||||
ftp_account ,
|
||||
ftp_url ,
|
||||
ftp_content ,
|
||||
bgp_type ,
|
||||
bgp_as_num ,
|
||||
bgp_route ,
|
||||
voip_calling_account ,
|
||||
voip_called_account ,
|
||||
voip_calling_number ,
|
||||
voip_called_number ,
|
||||
streaming_media_url ,
|
||||
streaming_media_protocol ,
|
||||
app_extra_info, common_vsys_id)
|
||||
select
|
||||
common_log_id,
|
||||
common_service ,
|
||||
common_recv_time ,
|
||||
common_direction ,
|
||||
common_l4_protocol ,
|
||||
common_address_type ,
|
||||
common_schema_type ,
|
||||
common_policy_id ,
|
||||
common_user_tags ,
|
||||
common_action ,
|
||||
common_sub_action ,
|
||||
common_user_region ,
|
||||
common_client_ip ,
|
||||
common_client_port ,
|
||||
common_internal_ip ,
|
||||
common_entrance_id ,
|
||||
common_device_id ,
|
||||
common_isp ,
|
||||
common_device_tag ,
|
||||
common_data_center ,
|
||||
common_encapsulation ,
|
||||
common_sled_ip ,
|
||||
common_client_location ,
|
||||
common_client_asn ,
|
||||
common_subscriber_id ,
|
||||
common_server_ip ,
|
||||
common_server_port ,
|
||||
common_external_ip ,
|
||||
common_server_location ,
|
||||
common_server_asn ,
|
||||
common_protocol_label ,
|
||||
common_app_label ,
|
||||
common_l7_protocol ,
|
||||
common_sessions ,
|
||||
common_c2s_pkt_num ,
|
||||
common_s2c_pkt_num ,
|
||||
common_c2s_byte_num ,
|
||||
common_s2c_byte_num ,
|
||||
common_start_time ,
|
||||
common_end_time ,
|
||||
common_establish_latency_ms ,
|
||||
common_con_duration_ms ,
|
||||
common_stream_dir ,
|
||||
common_address_list ,
|
||||
common_has_dup_traffic ,
|
||||
common_stream_error ,
|
||||
common_log_id ,
|
||||
common_link_info_c2s ,
|
||||
common_link_info_s2c ,
|
||||
common_c2s_ipfrag_num ,
|
||||
common_s2c_ipfrag_num ,
|
||||
common_c2s_tcp_lostlen ,
|
||||
common_s2c_tcp_lostlen ,
|
||||
common_c2s_tcp_unorder_num ,
|
||||
common_s2c_tcp_unorder_num ,
|
||||
common_first_ttl ,
|
||||
common_processing_time ,
|
||||
http_url ,
|
||||
http_host ,
|
||||
http_domain ,
|
||||
http_request_line ,
|
||||
http_response_line ,
|
||||
http_request_header ,
|
||||
http_response_header ,
|
||||
http_request_body ,
|
||||
http_response_body ,
|
||||
http_request_body_key ,
|
||||
http_response_body_key ,
|
||||
http_proxy_flag ,
|
||||
http_sequence ,
|
||||
http_snapshot ,
|
||||
http_cookie ,
|
||||
http_referer ,
|
||||
http_user_agent ,
|
||||
http_content_length ,
|
||||
http_content_type ,
|
||||
http_set_cookie ,
|
||||
http_version ,
|
||||
http_session_duration_ms ,
|
||||
http_action_file_size ,
|
||||
mail_protocol_type ,
|
||||
mail_account ,
|
||||
mail_to_cmd ,
|
||||
mail_from_cmd ,
|
||||
mail_from ,
|
||||
mail_to ,
|
||||
mail_cc ,
|
||||
mail_bcc ,
|
||||
mail_subject ,
|
||||
mail_subject_charset ,
|
||||
mail_content ,
|
||||
mail_content_charset ,
|
||||
mail_attachment_name ,
|
||||
mail_attachment_name_charset ,
|
||||
mail_attachment_content ,
|
||||
mail_eml_file ,
|
||||
mail_snapshot ,
|
||||
dns_message_id ,
|
||||
dns_qr ,
|
||||
dns_opcode ,
|
||||
dns_aa ,
|
||||
dns_tc ,
|
||||
dns_rd ,
|
||||
dns_ra ,
|
||||
dns_rcode ,
|
||||
dns_qdcount ,
|
||||
dns_ancount ,
|
||||
dns_nscount ,
|
||||
dns_arcount ,
|
||||
dns_qname ,
|
||||
dns_qtype ,
|
||||
dns_qclass ,
|
||||
dns_cname ,
|
||||
dns_sub ,
|
||||
dns_rr ,
|
||||
ssl_version ,
|
||||
ssl_sni ,
|
||||
ssl_san ,
|
||||
ssl_cn ,
|
||||
ssl_pinningst ,
|
||||
ssl_intercept_state ,
|
||||
ssl_server_side_latency ,
|
||||
ssl_client_side_latency ,
|
||||
ssl_server_side_version ,
|
||||
ssl_client_side_version ,
|
||||
ssl_cert_verify ,
|
||||
ssl_error ,
|
||||
ssl_con_latency_ms ,
|
||||
ssl_ja3_fingerprint ,
|
||||
ssl_ja3_hash ,
|
||||
quic_version ,
|
||||
quic_sni ,
|
||||
quic_user_agent ,
|
||||
ftp_account ,
|
||||
ftp_url ,
|
||||
ftp_content ,
|
||||
bgp_type ,
|
||||
bgp_as_num ,
|
||||
bgp_route ,
|
||||
voip_calling_account ,
|
||||
voip_called_account ,
|
||||
voip_calling_number ,
|
||||
voip_called_number ,
|
||||
streaming_media_url ,
|
||||
streaming_media_protocol ,
|
||||
app_extra_info,rand()%100 as common_vsys_id
|
||||
from
|
||||
$data_source_table
|
||||
where
|
||||
common_recv_time >= $input_time_start
|
||||
and common_recv_time < $input_time_end"
|
||||
start_time=`date -d @$input_time_start "+%Y-%m-%d %H:%M:%S"`
|
||||
end_time=`date -d @$input_time_end "+%Y-%m-%d %H:%M:%S"`
|
||||
echo " $data_destination_table common_recv_time>=$start_time and common_recv_time <=$end_time finished"
|
||||
done
|
||||
else
|
||||
echo " Error timestamp"
|
||||
continue
|
||||
fi
|
||||
echo " $ip done"
|
||||
done
|
||||
echo " Migrate Data Finished"
|
||||
3118
tsg_olap/installation/clickhouse/江苏项目/js-tsg-2402-create-table.sql
Normal file
3118
tsg_olap/installation/clickhouse/江苏项目/js-tsg-2402-create-table.sql
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
select 'session_record_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.session_record_old;
|
||||
|
||||
select 'security_event_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.security_event_old;
|
||||
|
||||
select 'transaction_record_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.transaction_record_old;
|
||||
|
||||
select 'voip_record_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.voip_record_old;
|
||||
|
||||
select 'proxy_event_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.proxy_event_old;
|
||||
|
||||
select 'dos_event_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.dos_event_old;
|
||||
@@ -0,0 +1,11 @@
|
||||
select 'session_record' as table_name, count(*) as cnt from tsg_galaxy_v3.session_record;
|
||||
|
||||
select 'security_event' as table_name, count(*) as cnt from tsg_galaxy_v3.security_event;
|
||||
|
||||
select 'transaction_record' as table_name, count(*) as cnt from tsg_galaxy_v3.transaction_record;
|
||||
|
||||
select 'voip_record' as table_name, count(*) as cnt from tsg_galaxy_v3.voip_record;
|
||||
|
||||
select 'proxy_event' as table_name, count(*) as cnt from tsg_galaxy_v3.proxy_event;
|
||||
|
||||
select 'dos_event' as table_name, count(*) as cnt from tsg_galaxy_v3.dos_event;
|
||||
@@ -0,0 +1,20 @@
|
||||
SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
|
||||
FROM tsg_galaxy_tmp.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT vsys_id, recv_time, log_id, profile_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, session_rate, packet_rate, bit_rate
|
||||
FROM tsg_galaxy_tmp.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_tmp.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_tmp.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_tmp.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_tmp.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
|
||||
FROM tsg_galaxy_tmp.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_tmp.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
# 遍历每个节点执行迁移
|
||||
for ip in `cat iplist.txt`
|
||||
do
|
||||
# 后台执行，输出日志
|
||||
echo "$ip节点"
|
||||
ssh $ip "[ ! -d $home ] && mkdir -p $home"
|
||||
scp -r $home/*local_table_to_2402.sh $ip:$home/
|
||||
ssh $ip "cd $home && chmod +x ./*.sh"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据，TSG24.01日志重组后数据迁移
|
||||
|
||||
# 迁移表 参数，可选值：session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00"
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00"
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数, 例如:240
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"session_record")
|
||||
# 迁移session_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"security_event")
|
||||
# 迁移security_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"monitor_event")
|
||||
# 迁移monitor_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"transaction_record")
|
||||
# 迁移transaction_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"voip_record")
|
||||
# 迁移voip_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"proxy_event")
|
||||
# 迁移proxy_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "slice_interval_minute参数必须是正确的分钟数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
# 遍历每个节点执行迁移
|
||||
for ip in `cat iplist.txt`
|
||||
do
|
||||
# 后台执行，输出日志
|
||||
echo "$ip 节点开始执行迁移"
|
||||
ssh $ip "cd $home && chmod +x ./*.sh && ./start_migrate_local_table_to_2402.sh $table '$data_start_time' '$data_end_time' $slice_interval_minute"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
table=$1
|
||||
if [ -z "$table" ]; then
|
||||
echo "缺少table参数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ips=($(cat iplist.txt))
|
||||
ips_size=${#ips[*]}
|
||||
ip_starts=$( seq 0 $(($ips_size - 1)) )
|
||||
ip_ends=$( seq 0 $(($ips_size - 1)) )
|
||||
|
||||
for ((i=0;i<$ips_size;i++))
|
||||
do
|
||||
ip_starts[$i]=0
|
||||
ip_ends[$i]=0
|
||||
done
|
||||
|
||||
while true ; do
|
||||
# 遍历每个节点
|
||||
for ((i=0;i<$ips_size;i++)); do
|
||||
ip=${ips[$i]}
|
||||
start=${ip_starts[$i]}
|
||||
end=${ip_ends[$i]}
|
||||
if [ $start -eq 0 ]; then
|
||||
info=$(ssh $ip "cat $home/log_$table.txt | grep migrate_table_start")
|
||||
if [ -n "$info" ]; then
|
||||
echo "${ip}迁移开始:${info}"
|
||||
ip_starts[$i]=1
|
||||
start=1
|
||||
fi
|
||||
fi
|
||||
if [ $start -eq 1 ] && [ $end -eq 0 ] ; then
|
||||
info=$(ssh $ip "cat $home/log_$table.txt | grep migrate_table_end")
|
||||
if [ -n "$info" ]; then
|
||||
echo "${ip}迁移结束:${info}"
|
||||
ip_ends[$i]=1
|
||||
end=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
#全部结束
|
||||
finish_cnt=0
|
||||
for ((i=0;i<$ips_size;i++)); do
|
||||
start=${ip_starts[$i]}
|
||||
end=${ip_ends[$i]}
|
||||
if [ $start -eq 1 ] && [ $end -eq 1 ] ; then
|
||||
finish_cnt=$(($finish_cnt+1))
|
||||
fi
|
||||
done
|
||||
if [ $finish_cnt -ge $ips_size ]; then
|
||||
echo "所有节点迁移结束"
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
@@ -0,0 +1 @@
|
||||
192.168.41.30
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据，TSG24.01日志重组后数据迁移
|
||||
|
||||
# 迁移表 参数，可选值：session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00"
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00"
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数, 例如:240
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"session_record")
|
||||
# 迁移session_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"security_event")
|
||||
# 迁移security_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"monitor_event")
|
||||
# 迁移monitor_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"transaction_record")
|
||||
# 迁移transaction_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"voip_record")
|
||||
# 迁移voip_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"proxy_event")
|
||||
# 迁移proxy_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "slice_interval_minute参数必须是正确的分钟数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 后台执行，输出日志
|
||||
nohup ./migrate_local_table_to_2402.sh "$table" "$data_start_time" "$data_end_time" $slice_interval_minute > "log_$table.txt" 2>&1 &
|
||||
echo "已启动迁移${table}表任务，时间范围[$data_start_time, $data_end_time], 每批迁移段分钟:$slice_interval_minute, 日志输出到:log_$table.txt。请查看日志文件确认每段数据迁移情况"
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- 删除源表同步子表物化视图
|
||||
|
||||
|
||||
-- 源表rename到历史表
|
||||
RENAME TABLE tsg_galaxy_tmp.session_record_local to tsg_galaxy_tmp.session_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.security_event_local to tsg_galaxy_tmp.security_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.transaction_record_local to tsg_galaxy_tmp.transaction_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.voip_record_local to tsg_galaxy_tmp.voip_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.proxy_event_local to tsg_galaxy_tmp.proxy_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.dos_event_local to tsg_galaxy_tmp.dos_event_local_old on cluster ck_cluster;
|
||||
|
||||
drop table if exists tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query;
|
||||
|
||||
|
||||
-- 创建源分布式表old
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local_old,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local_old,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query(
|
||||
log_id UInt64,
|
||||
profile_id UInt64,
|
||||
start_time Int64
|
||||
) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local_old,rand());
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- 删除源表同步子表物化视图
|
||||
create database if not exists tsg_galaxy_tmp on cluster ck_cluster;
|
||||
create database if not exists tsg_galaxy_tmp on cluster ck_query;
|
||||
|
||||
-- 源表rename到历史表
|
||||
RENAME TABLE tsg_galaxy_v3.session_record_local to tsg_galaxy_tmp.session_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.security_event_local to tsg_galaxy_tmp.security_event_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.transaction_record_local to tsg_galaxy_tmp.transaction_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.voip_record_local to tsg_galaxy_tmp.voip_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.proxy_event_local to tsg_galaxy_tmp.proxy_event_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.dos_event_local to tsg_galaxy_tmp.dos_event_local on cluster ck_cluster;
|
||||
|
||||
|
||||
-- 创建源分布式表old
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query(
|
||||
log_id UInt64,
|
||||
profile_id UInt64,
|
||||
start_time Int64
|
||||
) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local,rand());
|
||||
|
||||
321
tsg_olap/installation/clickhouse/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md
Normal file
321
tsg_olap/installation/clickhouse/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md
Normal file
@@ -0,0 +1,321 @@
|
||||
由于各环境当前使用tsg版本与升级的版本均不同,故在此提供通用步骤,因主键与字段不同,旧版本统一升级至23.07版本进行处理,为及时接入数据,离线迁移操作在tmp数据库进行。
|
||||
具体步骤：
|
||||
|
||||
Step1 ：停止入库任务。
|
||||
Step2 ：旧版本clickhouse库表迁移至tsg_galaxy_tmp,验证成功后删除tsg_galaxy_v3数据库
|
||||
Step3 : tsg_galaxy_v3数据库新建目标版本库表（如24.04）
|
||||
Step4 : 启动入库任务->tsg_galaxy_v3
|
||||
Step5 : tsg_galaxy_tmp数据库表升级到23.07修改升级语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp,升级验证成功后重命名为old表。
|
||||
Step6 ：tsg_galaxy_tmp数据库新建24.02版本库表,修改建表语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp
|
||||
Step7 : 迁移脚本迁移tsg_galaxy_tmp(23.07)->tsg_galaxy_tmp(24.02)
|
||||
Step8 : 升级tsg_galaxy_tmp(24.02)->升级tsg_galaxy_tmp目标版本如（24.04）
|
||||
Step9 : 命令迁移tsg_galaxy_tmp目标版本如（24.04）->tsg_galaxy_v3目标版本如（24.04）,按照（天）partition手动迁移
|
||||
|
||||
说明
|
||||
|
||||
请按步骤依次执行,执行脚本报错时联系研发处理后再执行之后的步骤。
|
||||
所有ck步骤都需要在query节点执行
|
||||
执行所有sql语句之前需要停止日志留存调度任务,确保ck中无分布式ddl语句执行,否则执行的sql会阻塞住,影响后续步骤执行
|
||||
验证sql需要在所有query节点执行
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1"
|
||||
若返回结果为空则可执行升级步骤,否则需要等待。
|
||||
|
||||
|
||||
一、停止旧表ck入库任务
|
||||
停止旧表ck入库任务
|
||||
|
||||
二、旧版本clickhouse库表迁移至tsg_galaxy_tmp
|
||||
|
||||
1.查看迁移前v3库表数据量
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_v3_old_table_row_count.sql
|
||||
|
||||
|
||||
2.执行迁移sql
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_v3_to_tmp.sql
|
||||
|
||||
|
||||
3.查看tmp库old表数据量
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_tmp_old_table_row_count.sql
|
||||
|
||||
4.数据量与原数据量一致,可删除数据库tsg_galaxy_v3。
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 --query "drop database if exists tsg_galaxy_v3 on cluster ck_cluster "
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 --query "drop database if exists tsg_galaxy_v3 on cluster ck_query "
|
||||
|
||||
三、tsg_galaxy_v3数据库新建目标版本库表并进行校验
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 对应版本初始化sql
|
||||
|
||||
四、启动ck入库任务
|
||||
|
||||
1.启动目标版本ck入库任务
|
||||
|
||||
|
||||
五、tsg_galaxy_tmp数据库表升级到23.07,升级验证成功后重命名为old表
|
||||
|
||||
1.tsg_galaxy_tmp旧版本clickhouse库表升级到23.07版本（注意所有版本升级sql需要将tsg_galaxy_v3替换tsg_galaxy_tmp,包括校验sql）,依次执行版本升级语句,并进行校验
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 各版本升级sql
|
||||
|
||||
2.验证成功后重命名为old表
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_tmp_2307_to_tmp_2307_old.sql
|
||||
|
||||
|
||||
3.查看tmp库old表数据量
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_tmp_old_table_row_count.sql
|
||||
|
||||
|
||||
六、临时库初始化24.02版本库表
|
||||
|
||||
1.执行2402版本初始化建表语句
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < init_tsg_galaxy_tmp_24_02_table.sql
|
||||
|
||||
2.校验表结构
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < check_tsg_galaxy_tmp_24_02_table.sql
|
||||
无报错信息说明校验通过
|
||||
|
||||
|
||||
七、离线脚本同步历史数据至临时数据库2402版本库表
|
||||
在query节点执行以下步骤,iplist.txt中为ck所有data节点ip地址。
|
||||
步骤描述：
|
||||
|
||||
1.进入migrate_table_2402文件夹,使脚本可执行
|
||||
|
||||
|
||||
chmod +x ./*.sh
|
||||
|
||||
|
||||
|
||||
2.分发迁移脚本到data节点
|
||||
|
||||
|
||||
./01_send_migrate_table_scripts.sh
|
||||
|
||||
|
||||
|
||||
2.选择迁移某个表,同步需要时间区间的数据,时间区间:实时同步任务开始时间向前推n天, 实时同步任务开始时间),时间区间为左闭右开,不包含结束时间点。
|
||||
|
||||
|
||||
# 迁移security_event表
|
||||
./02_start_migrate_table.sh security_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
|
||||
|
||||
|
||||
3.监控data节点迁移情况,所有表迁移完成后,确认每个节点同步数据成功/失败批次数,如有失败批次确认是否需要处理
|
||||
|
||||
|
||||
# 监控security_event表迁移
|
||||
./03_monitor_migrate_table.sh security_event
|
||||
|
||||
|
||||
|
||||
4.选择下个张需要迁移的表,重复2-4步骤。支持选择迁移的表有: security_event, monitor_event, session_record, transaction_record, voip_record, proxy_event, dos_event。
|
||||
|
||||
迁移和监控各个表执行命令示例：
|
||||
|
||||
# 迁移security_event表
|
||||
./02_start_migrate_table.sh security_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控security_event表迁移
|
||||
./03_monitor_migrate_table.sh security_event
|
||||
|
||||
|
||||
# 迁移monitor_event表
|
||||
./02_start_migrate_table.sh monitor_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控monitor_event表迁移
|
||||
./03_monitor_migrate_table.sh monitor_event
|
||||
|
||||
|
||||
# 迁移session_record表
|
||||
./02_start_migrate_table.sh session_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控session_record表迁移
|
||||
./03_monitor_migrate_table.sh session_record
|
||||
|
||||
|
||||
# 迁移transaction_record表
|
||||
./02_start_migrate_table.sh transaction_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控transaction_record表迁移
|
||||
./03_monitor_migrate_table.sh transaction_record
|
||||
|
||||
|
||||
# 迁移voip_record表
|
||||
./02_start_migrate_table.sh voip_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控voip_record表迁移
|
||||
./03_monitor_migrate_table.sh voip_record
|
||||
|
||||
|
||||
# 迁移proxy_event表
|
||||
./02_start_migrate_table.sh proxy_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控proxy_event表迁移
|
||||
./03_monitor_migrate_table.sh proxy_event
|
||||
|
||||
|
||||
# 迁移dos_event表
|
||||
./02_start_migrate_table.sh dos_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控dos_event表迁移
|
||||
./03_monitor_migrate_table.sh dos_event
|
||||
|
||||
|
||||
迁移日志无报错,数据迁移完成。
|
||||
如果有数据迁移失败批次,查看新老表迁移数据量对应情况(ck每台data节点)：
|
||||
|
||||
-- security_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.security_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
and common_action in (16, 96)
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.security_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- monitor_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.security_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
and common_action = 1
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.monitor_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- session_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.session_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.session_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- transaction_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.transaction_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.transaction_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- voip_record
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.voip_record_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.voip_record_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- proxy_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(common_recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.proxy_event_local_old
|
||||
WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(common_recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.proxy_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- dos_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(start_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.dos_event_local_old
|
||||
WHERE start_time>= toUnixTimestamp('2024-01-10 00:00:00') and start_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(start_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(start_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.dos_event_local
|
||||
WHERE start_time >= toUnixTimestamp('2024-01-10 00:00:00') and start_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(start_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
|
||||
八、升级tsg_galaxy_tmp(24.02)->升级tsg_galaxy_tmp目标版本如（24.04）主要目的使临时库与实际库表结构相同
|
||||
|
||||
从24.03开始依次执行直至目标版本（注意所有版本升级sql需要将tsg_galaxy_v3替换tsg_galaxy_tmp,包括校验sql）
|
||||
|
||||
|
||||
九、命令迁移tsg_galaxy_tmp目标版本如（24.04）->tsg_galaxy_v3目标版本如（24.04）,按照（天）partition手动迁移
|
||||
|
||||
登录命令行之后手动按需执行需要迁移的分区（如下示例迁移2024年4月1日的数据）
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ******
|
||||
|
||||
ALTER TABLE tsg_galaxy_tmp.session_record_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.session_record_local;
|
||||
ALTER TABLE tsg_galaxy_tmp.security_event_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.security_event_local;
|
||||
ALTER TABLE tsg_galaxy_tmp.transaction_record_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.transaction_record_local;
|
||||
ALTER TABLE tsg_galaxy_tmp.voip_record_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.voip_record_local;
|
||||
ALTER TABLE tsg_galaxy_tmp.proxy_event_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.proxy_event_local;
|
||||
ALTER TABLE tsg_galaxy_tmp.dos_event_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.dos_event_local;
|
||||
Reference in New Issue
Block a user