diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/cat_tmp_old_table_row_count.sql b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/cat_tmp_old_table_row_count.sql new file mode 100644 index 0000000..87dd4ea --- /dev/null +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/cat_tmp_old_table_row_count.sql @@ -0,0 +1,11 @@ +select 'session_record_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.session_record_old; + +select 'security_event_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.security_event_old; + +select 'transaction_record_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.transaction_record_old; + +select 'voip_record_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.voip_record_old; + +select 'proxy_event_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.proxy_event_old; + +select 'dos_event_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.dos_event_old; \ No newline at end of file diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/check_tsg_galaxy_tmp_24_02_table.sql b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/check_tsg_galaxy_tmp_24_02_table.sql new file mode 100644 index 0000000..096e4cd --- /dev/null +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/check_tsg_galaxy_tmp_24_02_table.sql @@ -0,0 +1,20 @@ +SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha +FROM tsg_galaxy_tmp.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time $session_record_table_dest where common_recv_time >= $input_time_start and common_recv_time < $input_time_end" + clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql" + rst_code=$? + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + if [ $rst_code -eq 0 ]; then + insert_success_cnt=$(($insert_success_cnt+1)) + echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + else + insert_fail_cnt=$(($insert_fail_cnt+1)) + echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + fi +} + +migrate_security_event_table_one_part(){ + input_time_start=$1 + input_time_end=$2 + insert_sql="insert into $security_event_table_dest +( + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + -- proxy_rule_list, + -- statistics_rule_list, + -- sc_rule_list, + -- sc_rsp_raw, + -- sc_rsp_decrypted, + -- proxy_action, + -- proxy_pinning_status, + -- proxy_intercept_status, + -- proxy_passthrough_reason, + -- proxy_client_side_latency_ms, + -- proxy_server_side_latency_ms, + -- proxy_client_side_version, + -- proxy_server_side_version, + -- proxy_cert_verify, + -- proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + -- client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + -- apn, + server_ip, + server_port, + -- server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + -- app_debug_info, + -- app_content, + fqdn_category_list, + -- ip_protocol, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + -- http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + -- ssl_esni_flag, + -- ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + -- mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + -- stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +) +SELECT + common_recv_time as recv_time, + common_log_id as log_id, + common_schema_type as decoded_as, + common_stream_trace_id as session_id, + common_start_timestamp_ms as start_timestamp_ms, + common_end_timestamp_ms as end_timestamp_ms, + common_con_duration_ms as duration_ms, + common_establish_latency_ms as tcp_handshake_latency_ms, + common_ingestion_time as ingestion_time, + common_processing_time as processing_time, + -- common_insert_time as insert_time, + common_device_id as device_id, + common_egress_link_id as out_link_id, + common_ingress_link_id as in_link_id, + common_device_tag as device_tag, + common_data_center as data_center, + common_device_group as device_group, + common_sled_ip as sled_ip, + common_address_type as address_type, + common_vsys_id as vsys_id, + common_t_vsys_id as t_vsys_id, + common_flags as flags, + common_flags_identify_info as flags_identify_info, + if(common_action in (16, 96), array(common_policy_id), array()) as security_rule_list, -- in(Deny、Allow) array(common_policy_id) 安全表 + case common_action + when 16 then 'Deny' + when 96 then 'Allow' + else '' + end as security_action, -- Deny16、Allow96 else '' + if(common_action = 1, array(common_policy_id), array()) as monitor_rule_list, -- 是Monitor(1) 放common_policy_id + common_shaping_rule_ids as shaping_rule_list, + -- as proxy_rule_list, + -- as statistics_rule_list, + -- as sc_rule_list, + -- as sc_rsp_raw, + -- as sc_rsp_decrypted, + -- as proxy_action, + -- as proxy_pinning_status, + -- as proxy_intercept_status, + -- as proxy_passthrough_reason, + -- as proxy_client_side_latency_ms, + -- as proxy_server_side_latency_ms, + -- as proxy_client_side_version, + -- as proxy_server_side_version, + -- as proxy_cert_verify, + -- as proxy_intercept_error, + common_mirrored_pkts as monitor_mirrored_pkts, + common_mirrored_bytes as monitor_mirrored_bytes, + common_client_ip as client_ip, + common_client_port as client_port, + -- common_client_os_name as client_os_desc, + common_client_location as client_geolocation, + common_client_asn as client_asn, + common_subscriber_id as subscriber_id, + common_imei as imei, + common_imsi as imsi, + common_phone_number as phone_number, + -- as apn, + common_server_ip as server_ip, + common_server_port as server_port, + -- common_server_os_name as server_os_desc, + common_server_location as server_geolocation, + common_server_asn as server_asn, + common_server_fqdn as server_fqdn, + common_server_domain as server_domain, + common_app_full_path as app_transition, + common_app_label as app, + -- app_debug_info, + -- common_app_behavior as app_content, + common_service_category as fqdn_category_list, + -- ip_protocol, + common_protocol_label as decoded_path, + dns_message_id as dns_message_id, + dns_qr as dns_qr, + dns_opcode as dns_opcode, + dns_aa as dns_aa, + dns_tc as dns_tc, + dns_rd as dns_rd, + dns_ra as dns_ra, + dns_rcode as dns_rcode, + dns_qdcount as dns_qdcount, + dns_ancount as dns_ancount, + dns_nscount as dns_nscount, + dns_arcount as dns_arcount, + dns_qname as dns_qname, + dns_qtype as dns_qtype, + dns_qclass as dns_qclass, + dns_cname as dns_cname, + dns_sub as dns_sub, + dns_rr as dns_rr, + dns_response_latency_ms as dns_response_latency_ms, + http_url as http_url, + http_host as http_host, + http_request_line as http_request_line, + http_response_line as http_response_line, + http_request_body as http_request_body, + http_response_body as http_response_body, + http_proxy_flag as http_proxy_flag, + http_sequence as http_sequence, + http_cookie as http_cookie, + http_referer as http_referer, + http_user_agent as http_user_agent, + http_request_content_length as http_request_content_length, + http_request_content_type as http_request_content_type, + http_response_content_length as http_response_content_length, + http_response_content_type as http_response_content_type, + http_set_cookie as http_set_cookie, + http_version as http_version, + -- http_status_code as http_status_code, + http_response_latency_ms as http_response_latency_ms, + http_session_duration_ms as http_session_duration_ms, + http_action_file_size as http_action_file_size, + ssl_version as ssl_version, + ssl_sni as ssl_sni, + ssl_san as ssl_san, + ssl_cn as ssl_cn, + ssl_con_latency_ms as ssl_handshake_latency_ms, + ssl_ja3_hash as ssl_ja3_hash, + ssl_ja3s_hash as ssl_ja3s_hash, + ssl_cert_issuer as ssl_cert_issuer, + ssl_cert_subject as ssl_cert_subject, + -- ssl_esni_flag as ssl_esni_flag, + -- ssl_ech_flag as ssl_ech_flag, + dtls_cookie as dtls_cookie, + dtls_version as dtls_version, + dtls_sni as dtls_sni, + dtls_san as dtls_san, + dtls_cn as dtls_cn, + dtls_con_latency_ms as dtls_handshake_latency_ms, + dtls_ja3_fingerprint as dtls_ja3_fingerprint, + dtls_ja3_hash as dtls_ja3_hash, + dtls_cert_issuer as dtls_cert_issuer, + dtls_cert_subject as dtls_cert_subject, + mail_protocol_type as mail_protocol_type, + mail_account as mail_account, + mail_from_cmd as mail_from_cmd, + mail_to_cmd as mail_to_cmd, + mail_from as mail_from, + -- as mail_password, + mail_to as mail_to, + mail_cc as mail_cc, + mail_bcc as mail_bcc, + mail_subject as mail_subject, + mail_subject_charset as mail_subject_charset, + mail_attachment_name as mail_attachment_name, + mail_attachment_name_charset as mail_attachment_name_charset, + mail_eml_file as mail_eml_file, + ftp_account as ftp_account, + ftp_url as ftp_url, + ftp_link_type as ftp_link_type, + quic_version as quic_version, + quic_sni as quic_sni, + quic_user_agent as quic_user_agent, + rdp_cookie as rdp_cookie, + rdp_security_protocol as rdp_security_protocol, + rdp_client_channels as rdp_client_channels, + rdp_keyboard_layout as rdp_keyboard_layout, + rdp_client_version as rdp_client_version, + rdp_client_name as rdp_client_name, + rdp_client_product_id as rdp_client_product_id, + rdp_desktop_width as rdp_desktop_width, + rdp_desktop_height as rdp_desktop_height, + rdp_requested_color_depth as rdp_requested_color_depth, + rdp_certificate_type as rdp_certificate_type, + rdp_certificate_count as rdp_certificate_count, + rdp_certificate_permanent as rdp_certificate_permanent, + rdp_encryption_level as rdp_encryption_level, + rdp_encryption_method as rdp_encryption_method, + ssh_version as ssh_version, + ssh_auth_success as ssh_auth_success, + ssh_client_version as ssh_client_version, + ssh_server_version as ssh_server_version, + ssh_cipher_alg as ssh_cipher_alg, + ssh_mac_alg as ssh_mac_alg, + ssh_compression_alg as ssh_compression_alg, + ssh_kex_alg as ssh_kex_alg, + ssh_host_key_alg as ssh_host_key_alg, + ssh_host_key as ssh_host_key, + ssh_hassh as ssh_hassh, + sip_call_id as sip_call_id, + sip_originator_description as sip_originator_description, + sip_responder_description as sip_responder_description, + sip_user_agent as sip_user_agent, + sip_server as sip_server, + sip_originator_sdp_connect_ip as sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port as sip_originator_sdp_media_port, + sip_originator_sdp_media_type as sip_originator_sdp_media_type, + sip_originator_sdp_content as sip_originator_sdp_content, + sip_responder_sdp_connect_ip as sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port as sip_responder_sdp_media_port, + sip_responder_sdp_media_type as sip_responder_sdp_media_type, + sip_responder_sdp_content as sip_responder_sdp_content, + sip_duration_s as sip_duration_s, + sip_bye as sip_bye, + rtp_payload_type_c2s as rtp_payload_type_c2s, + rtp_payload_type_s2c as rtp_payload_type_s2c, + rtp_pcap_path as rtp_pcap_path, + rtp_originator_dir as rtp_originator_dir, + stratum_cryptocurrency as stratum_cryptocurrency, + stratum_mining_pools as stratum_mining_pools, + stratum_mining_program as stratum_mining_program, + -- as stratum_mining_subscribe, + common_c2s_pkt_num as sent_pkts, + common_s2c_pkt_num as received_pkts, + common_c2s_byte_num as sent_bytes, + common_s2c_byte_num as received_bytes, + common_c2s_ipfrag_num as tcp_c2s_ip_fragments, + common_s2c_ipfrag_num as tcp_s2c_ip_fragments, + common_c2s_tcp_lostlen as tcp_c2s_lost_bytes, + common_s2c_tcp_lostlen as tcp_s2c_lost_bytes, + common_c2s_tcp_unorder_num as tcp_c2s_o3_pkts, + common_s2c_tcp_unorder_num as tcp_s2c_o3_pkts, + common_c2s_pkt_retrans as tcp_c2s_rtx_pkts, + common_s2c_pkt_retrans as tcp_s2c_rtx_pkts, + common_c2s_byte_retrans as tcp_c2s_rtx_bytes, + common_s2c_byte_retrans as tcp_s2c_rtx_bytes, + common_first_ttl as tcp_rtt_ms, + common_tcp_client_isn as tcp_client_isn, + common_tcp_server_isn as tcp_server_isn, + common_packet_capture_file as packet_capture_file, + common_in_src_mac as in_src_mac, + common_out_src_mac as out_src_mac, + common_in_dest_mac as in_dest_mac, + common_out_dest_mac as out_dest_mac, + common_tunnels as encapsulation, + common_has_dup_traffic as dup_traffic_flag, + common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, + common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc +FROM $security_event_table_src +where common_recv_time >= $input_time_start and common_recv_time < $input_time_end + and common_action in (16, 96) +;" + # echo "sql:$security_event_table_src => $security_event_table_dest where common_recv_time >= $input_time_start and common_recv_time < $input_time_end" + clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql" + rst_code=$? + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + if [ $rst_code -eq 0 ]; then + insert_success_cnt=$(($insert_success_cnt+1)) + echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + else + insert_fail_cnt=$(($insert_fail_cnt+1)) + echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + fi +} + +migrate_monitor_event_table_one_part(){ + input_time_start=$1 + input_time_end=$2 + insert_sql="insert into $monitor_event_table_dest +( + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + -- proxy_rule_list, + -- statistics_rule_list, + -- sc_rule_list, + -- sc_rsp_raw, + -- sc_rsp_decrypted, + -- proxy_action, + -- proxy_pinning_status, + -- proxy_intercept_status, + -- proxy_passthrough_reason, + -- proxy_client_side_latency_ms, + -- proxy_server_side_latency_ms, + -- proxy_client_side_version, + -- proxy_server_side_version, + -- proxy_cert_verify, + -- proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + -- client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + -- apn, + server_ip, + server_port, + -- server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + -- app_debug_info, + -- app_content, + fqdn_category_list, + -- ip_protocol, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + -- http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + -- ssl_esni_flag, + -- ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + -- mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + -- stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +) +SELECT + common_recv_time as recv_time, + common_log_id as log_id, + common_schema_type as decoded_as, + common_stream_trace_id as session_id, + common_start_timestamp_ms as start_timestamp_ms, + common_end_timestamp_ms as end_timestamp_ms, + common_con_duration_ms as duration_ms, + common_establish_latency_ms as tcp_handshake_latency_ms, + common_ingestion_time as ingestion_time, + common_processing_time as processing_time, + -- common_insert_time as insert_time, + common_device_id as device_id, + common_egress_link_id as out_link_id, + common_ingress_link_id as in_link_id, + common_device_tag as device_tag, + common_data_center as data_center, + common_device_group as device_group, + common_sled_ip as sled_ip, + common_address_type as address_type, + common_vsys_id as vsys_id, + common_t_vsys_id as t_vsys_id, + common_flags as flags, + common_flags_identify_info as flags_identify_info, + if(common_action in (16, 96), array(common_policy_id), array()) as security_rule_list, -- in(Deny、Allow) array(common_policy_id) 安全表 + case common_action + when 16 then 'Deny' + when 96 then 'Allow' + else '' + end as security_action, -- Deny16、Allow96 else '' + if(common_action = 1, array(common_policy_id), array()) as monitor_rule_list, -- 是Monitor(1) 放common_policy_id + common_shaping_rule_ids as shaping_rule_list, + -- as proxy_rule_list, + -- as statistics_rule_list, + -- as sc_rule_list, + -- as sc_rsp_raw, + -- as sc_rsp_decrypted, + -- as proxy_action, + -- as proxy_pinning_status, + -- as proxy_intercept_status, + -- as proxy_passthrough_reason, + -- as proxy_client_side_latency_ms, + -- as proxy_server_side_latency_ms, + -- as proxy_client_side_version, + -- as proxy_server_side_version, + -- as proxy_cert_verify, + -- as proxy_intercept_error, + common_mirrored_pkts as monitor_mirrored_pkts, + common_mirrored_bytes as monitor_mirrored_bytes, + common_client_ip as client_ip, + common_client_port as client_port, + -- common_client_os_name as client_os_desc, + common_client_location as client_geolocation, + common_client_asn as client_asn, + common_subscriber_id as subscriber_id, + common_imei as imei, + common_imsi as imsi, + common_phone_number as phone_number, + -- as apn, + common_server_ip as server_ip, + common_server_port as server_port, + -- common_server_os_name as server_os_desc, + common_server_location as server_geolocation, + common_server_asn as server_asn, + common_server_fqdn as server_fqdn, + common_server_domain as server_domain, + common_app_full_path as app_transition, + common_app_label as app, + -- app_debug_info, + -- common_app_behavior as app_content, + common_service_category as fqdn_category_list, + -- ip_protocol, + common_protocol_label as decoded_path, + dns_message_id as dns_message_id, + dns_qr as dns_qr, + dns_opcode as dns_opcode, + dns_aa as dns_aa, + dns_tc as dns_tc, + dns_rd as dns_rd, + dns_ra as dns_ra, + dns_rcode as dns_rcode, + dns_qdcount as dns_qdcount, + dns_ancount as dns_ancount, + dns_nscount as dns_nscount, + dns_arcount as dns_arcount, + dns_qname as dns_qname, + dns_qtype as dns_qtype, + dns_qclass as dns_qclass, + dns_cname as dns_cname, + dns_sub as dns_sub, + dns_rr as dns_rr, + dns_response_latency_ms as dns_response_latency_ms, + http_url as http_url, + http_host as http_host, + http_request_line as http_request_line, + http_response_line as http_response_line, + http_request_body as http_request_body, + http_response_body as http_response_body, + http_proxy_flag as http_proxy_flag, + http_sequence as http_sequence, + http_cookie as http_cookie, + http_referer as http_referer, + http_user_agent as http_user_agent, + http_request_content_length as http_request_content_length, + http_request_content_type as http_request_content_type, + http_response_content_length as http_response_content_length, + http_response_content_type as http_response_content_type, + http_set_cookie as http_set_cookie, + http_version as http_version, + -- http_status_code as http_status_code, + http_response_latency_ms as http_response_latency_ms, + http_session_duration_ms as http_session_duration_ms, + http_action_file_size as http_action_file_size, + ssl_version as ssl_version, + ssl_sni as ssl_sni, + ssl_san as ssl_san, + ssl_cn as ssl_cn, + ssl_con_latency_ms as ssl_handshake_latency_ms, + ssl_ja3_hash as ssl_ja3_hash, + ssl_ja3s_hash as ssl_ja3s_hash, + ssl_cert_issuer as ssl_cert_issuer, + ssl_cert_subject as ssl_cert_subject, + -- ssl_esni_flag as ssl_esni_flag, + -- ssl_ech_flag as ssl_ech_flag, + dtls_cookie as dtls_cookie, + dtls_version as dtls_version, + dtls_sni as dtls_sni, + dtls_san as dtls_san, + dtls_cn as dtls_cn, + dtls_con_latency_ms as dtls_handshake_latency_ms, + dtls_ja3_fingerprint as dtls_ja3_fingerprint, + dtls_ja3_hash as dtls_ja3_hash, + dtls_cert_issuer as dtls_cert_issuer, + dtls_cert_subject as dtls_cert_subject, + mail_protocol_type as mail_protocol_type, + mail_account as mail_account, + mail_from_cmd as mail_from_cmd, + mail_to_cmd as mail_to_cmd, + mail_from as mail_from, + -- as mail_password, + mail_to as mail_to, + mail_cc as mail_cc, + mail_bcc as mail_bcc, + mail_subject as mail_subject, + mail_subject_charset as mail_subject_charset, + mail_attachment_name as mail_attachment_name, + mail_attachment_name_charset as mail_attachment_name_charset, + mail_eml_file as mail_eml_file, + ftp_account as ftp_account, + ftp_url as ftp_url, + ftp_link_type as ftp_link_type, + quic_version as quic_version, + quic_sni as quic_sni, + quic_user_agent as quic_user_agent, + rdp_cookie as rdp_cookie, + rdp_security_protocol as rdp_security_protocol, + rdp_client_channels as rdp_client_channels, + rdp_keyboard_layout as rdp_keyboard_layout, + rdp_client_version as rdp_client_version, + rdp_client_name as rdp_client_name, + rdp_client_product_id as rdp_client_product_id, + rdp_desktop_width as rdp_desktop_width, + rdp_desktop_height as rdp_desktop_height, + rdp_requested_color_depth as rdp_requested_color_depth, + rdp_certificate_type as rdp_certificate_type, + rdp_certificate_count as rdp_certificate_count, + rdp_certificate_permanent as rdp_certificate_permanent, + rdp_encryption_level as rdp_encryption_level, + rdp_encryption_method as rdp_encryption_method, + ssh_version as ssh_version, + ssh_auth_success as ssh_auth_success, + ssh_client_version as ssh_client_version, + ssh_server_version as ssh_server_version, + ssh_cipher_alg as ssh_cipher_alg, + ssh_mac_alg as ssh_mac_alg, + ssh_compression_alg as ssh_compression_alg, + ssh_kex_alg as ssh_kex_alg, + ssh_host_key_alg as ssh_host_key_alg, + ssh_host_key as ssh_host_key, + ssh_hassh as ssh_hassh, + sip_call_id as sip_call_id, + sip_originator_description as sip_originator_description, + sip_responder_description as sip_responder_description, + sip_user_agent as sip_user_agent, + sip_server as sip_server, + sip_originator_sdp_connect_ip as sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port as sip_originator_sdp_media_port, + sip_originator_sdp_media_type as sip_originator_sdp_media_type, + sip_originator_sdp_content as sip_originator_sdp_content, + sip_responder_sdp_connect_ip as sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port as sip_responder_sdp_media_port, + sip_responder_sdp_media_type as sip_responder_sdp_media_type, + sip_responder_sdp_content as sip_responder_sdp_content, + sip_duration_s as sip_duration_s, + sip_bye as sip_bye, + rtp_payload_type_c2s as rtp_payload_type_c2s, + rtp_payload_type_s2c as rtp_payload_type_s2c, + rtp_pcap_path as rtp_pcap_path, + rtp_originator_dir as rtp_originator_dir, + stratum_cryptocurrency as stratum_cryptocurrency, + stratum_mining_pools as stratum_mining_pools, + stratum_mining_program as stratum_mining_program, + -- as stratum_mining_subscribe, + common_c2s_pkt_num as sent_pkts, + common_s2c_pkt_num as received_pkts, + common_c2s_byte_num as sent_bytes, + common_s2c_byte_num as received_bytes, + common_c2s_ipfrag_num as tcp_c2s_ip_fragments, + common_s2c_ipfrag_num as tcp_s2c_ip_fragments, + common_c2s_tcp_lostlen as tcp_c2s_lost_bytes, + common_s2c_tcp_lostlen as tcp_s2c_lost_bytes, + common_c2s_tcp_unorder_num as tcp_c2s_o3_pkts, + common_s2c_tcp_unorder_num as tcp_s2c_o3_pkts, + common_c2s_pkt_retrans as tcp_c2s_rtx_pkts, + common_s2c_pkt_retrans as tcp_s2c_rtx_pkts, + common_c2s_byte_retrans as tcp_c2s_rtx_bytes, + common_s2c_byte_retrans as tcp_s2c_rtx_bytes, + common_first_ttl as tcp_rtt_ms, + common_tcp_client_isn as tcp_client_isn, + common_tcp_server_isn as tcp_server_isn, + common_packet_capture_file as packet_capture_file, + common_in_src_mac as in_src_mac, + common_out_src_mac as out_src_mac, + common_in_dest_mac as in_dest_mac, + common_out_dest_mac as out_dest_mac, + common_tunnels as encapsulation, + common_has_dup_traffic as dup_traffic_flag, + common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, + common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc +FROM $monitor_event_table_src +where common_recv_time >= $input_time_start and common_recv_time < $input_time_end + and common_action = 1 +;" + # echo "sql:$monitor_event_table_src => $monitor_event_table_dest where common_recv_time >= $input_time_start and common_recv_time < $input_time_end" + clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql" + rst_code=$? + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + if [ $rst_code -eq 0 ]; then + insert_success_cnt=$(($insert_success_cnt+1)) + echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + else + insert_fail_cnt=$(($insert_fail_cnt+1)) + echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + fi +} + +migrate_transaction_record_table_one_part(){ + input_time_start=$1 + input_time_end=$2 + insert_sql="insert into $transaction_record_table_dest +( + recv_time, + log_id, + decoded_as, + session_id, + ingestion_time, + processing_time, + -- insert_time, + address_type, + vsys_id, + client_ip, + client_port, + server_ip, + server_port, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + -- http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + -- mail_protocol_type, + -- mail_account, + -- mail_from_cmd, + -- mail_to_cmd, + -- mail_from, + -- mail_password, + -- mail_to, + -- mail_cc, + -- mail_bcc, + -- mail_subject, + -- mail_subject_charset, + -- mail_attachment_name, + -- mail_attachment_name_charset, + -- mail_eml_file, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye +) +SELECT + common_recv_time as recv_time, + common_log_id as log_id, + common_schema_type as decoded_as, + common_stream_trace_id as session_id, + common_ingestion_time as ingestion_time, + common_processing_time as processing_time, + -- common_insert_time as insert_time, + common_address_type as address_type, + common_vsys_id as vsys_id, + common_client_ip as client_ip, + common_client_port as client_port, + common_server_ip as server_ip, + common_server_port as server_port, + common_c2s_pkt_num as sent_pkts, + common_s2c_pkt_num as received_pkts, + common_c2s_byte_num as sent_bytes, + common_s2c_byte_num as received_bytes, + dns_message_id as dns_message_id, + dns_qr as dns_qr, + dns_opcode as dns_opcode, + dns_aa as dns_aa, + dns_tc as dns_tc, + dns_rd as dns_rd, + dns_ra as dns_ra, + dns_rcode as dns_rcode, + dns_qdcount as dns_qdcount, + dns_ancount as dns_ancount, + dns_nscount as dns_nscount, + dns_arcount as dns_arcount, + dns_qname as dns_qname, + dns_qtype as dns_qtype, + dns_qclass as dns_qclass, + dns_cname as dns_cname, + dns_sub as dns_sub, + dns_rr as dns_rr, + dns_response_latency_ms as dns_response_latency_ms, + http_url as http_url, + http_host as http_host, + http_request_line as http_request_line, + http_response_line as http_response_line, + http_request_body as http_request_body, + http_response_body as http_response_body, + http_proxy_flag as http_proxy_flag, + http_sequence as http_sequence, + http_cookie as http_cookie, + http_referer as http_referer, + http_user_agent as http_user_agent, + http_request_content_length as http_request_content_length, + http_request_content_type as http_request_content_type, + http_response_content_length as http_response_content_length, + http_response_content_type as http_response_content_type, + http_set_cookie as http_set_cookie, + http_version as http_version, + -- http_status_code as http_status_code, + http_response_latency_ms as http_response_latency_ms, + http_session_duration_ms as http_session_duration_ms, + http_action_file_size as http_action_file_size, + -- as mail_protocol_type, + -- as mail_account, + -- as mail_from_cmd, + -- as mail_to_cmd, + -- as mail_from, + -- as mail_password, + -- as mail_to, + -- as mail_cc, + -- as mail_bcc, + -- as mail_subject, + -- as mail_subject_charset, + -- as mail_attachment_name, + -- as mail_attachment_name_charset, + -- as mail_eml_file, + sip_call_id as sip_call_id, + sip_originator_description as sip_originator_description, + sip_responder_description as sip_responder_description, + sip_user_agent as sip_user_agent, + sip_server as sip_server, + sip_originator_sdp_connect_ip as sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port as sip_originator_sdp_media_port, + sip_originator_sdp_media_type as sip_originator_sdp_media_type, + sip_originator_sdp_content as sip_originator_sdp_content, + sip_responder_sdp_connect_ip as sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port as sip_responder_sdp_media_port, + sip_responder_sdp_media_type as sip_responder_sdp_media_type, + sip_responder_sdp_content as sip_responder_sdp_content, + sip_duration_s as sip_duration_s, + sip_bye as sip_bye +FROM $transaction_record_table_src +where common_recv_time >= $input_time_start and common_recv_time < $input_time_end +;" + # echo "sql:$transaction_record_table_src => $transaction_record_table_dest where common_recv_time >= $input_time_start and common_recv_time < $input_time_end" + clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql" + rst_code=$? + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + if [ $rst_code -eq 0 ]; then + insert_success_cnt=$(($insert_success_cnt+1)) + echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + else + insert_fail_cnt=$(($insert_fail_cnt+1)) + echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + fi +} + +migrate_voip_record_table_one_part(){ + input_time_start=$1 + input_time_end=$2 + insert_sql="insert into $voip_record_table_dest +( + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + -- security_rule_list, + -- security_action, + -- monitor_rule_list, + shaping_rule_list, + -- proxy_rule_list, + -- statistics_rule_list, + -- sc_rule_list, + -- sc_rsp_raw, + -- sc_rsp_decrypted, + -- proxy_action, + -- proxy_pinning_status, + -- proxy_intercept_status, + -- proxy_passthrough_reason, + -- proxy_client_side_latency_ms, + -- proxy_server_side_latency_ms, + -- proxy_client_side_version, + -- proxy_server_side_version, + -- proxy_cert_verify, + -- proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + -- client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + -- apn, + server_ip, + server_port, + -- server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + -- app_debug_info, + -- app_content, + fqdn_category_list, + -- ip_protocol, + decoded_path, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +) +SELECT + common_recv_time as recv_time, + common_log_id as log_id, + common_schema_type as decoded_as, + common_stream_trace_id as session_id, + common_start_timestamp_ms as start_timestamp_ms, + common_end_timestamp_ms as end_timestamp_ms, + common_con_duration_ms as duration_ms, + common_establish_latency_ms as tcp_handshake_latency_ms, + common_ingestion_time as ingestion_time, + common_processing_time as processing_time, + -- common_insert_time as insert_time, + common_device_id as device_id, + common_egress_link_id as out_link_id, + common_ingress_link_id as in_link_id, + common_device_tag as device_tag, + common_data_center as data_center, + common_device_group as device_group, + common_sled_ip as sled_ip, + common_address_type as address_type, + common_vsys_id as vsys_id, + common_t_vsys_id as t_vsys_id, + common_flags as flags, + common_flags_identify_info as flags_identify_info, + -- as security_rule_list, + -- as security_action, + -- as monitor_rule_list, + common_shaping_rule_ids as shaping_rule_list, + -- as proxy_rule_list, + -- as statistics_rule_list, + -- as sc_rule_list, + -- as sc_rsp_raw, + -- as sc_rsp_decrypted, + -- as proxy_action, + -- as proxy_pinning_status, + -- as proxy_intercept_status, + -- as proxy_passthrough_reason, + -- as proxy_client_side_latency_ms, + -- as proxy_server_side_latency_ms, + -- as proxy_client_side_version, + -- as proxy_server_side_version, + -- as proxy_cert_verify, + -- as proxy_intercept_error, + common_mirrored_pkts as monitor_mirrored_pkts, + common_mirrored_bytes as monitor_mirrored_bytes, + common_client_ip as client_ip, + common_client_port as client_port, + -- common_client_os_name as client_os_desc, + common_client_location as client_geolocation, + common_client_asn as client_asn, + common_subscriber_id as subscriber_id, + common_imei as imei, + common_imsi as imsi, + common_phone_number as phone_number, + -- as apn, + common_server_ip as server_ip, + common_server_port as server_port, + -- common_server_os_name as server_os_desc, + common_server_location as server_geolocation, + common_server_asn as server_asn, + common_server_fqdn as server_fqdn, + common_server_domain as server_domain, + common_app_full_path as app_transition, + common_app_label as app, + -- app_debug_info, + -- common_app_behavior as app_content, + common_service_category as fqdn_category_list, + -- ip_protocol, + common_protocol_label as decoded_path, + sip_call_id as sip_call_id, + sip_originator_description as sip_originator_description, + sip_responder_description as sip_responder_description, + sip_user_agent as sip_user_agent, + sip_server as sip_server, + sip_originator_sdp_connect_ip as sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port as sip_originator_sdp_media_port, + sip_originator_sdp_media_type as sip_originator_sdp_media_type, + sip_originator_sdp_content as sip_originator_sdp_content, + sip_responder_sdp_connect_ip as sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port as sip_responder_sdp_media_port, + sip_responder_sdp_media_type as sip_responder_sdp_media_type, + sip_responder_sdp_content as sip_responder_sdp_content, + sip_duration_s as sip_duration_s, + sip_bye as sip_bye, + rtp_payload_type_c2s as rtp_payload_type_c2s, + rtp_payload_type_s2c as rtp_payload_type_s2c, + rtp_pcap_path as rtp_pcap_path, + rtp_originator_dir as rtp_originator_dir, + common_c2s_pkt_num as sent_pkts, + common_s2c_pkt_num as received_pkts, + common_c2s_byte_num as sent_bytes, + common_s2c_byte_num as received_bytes, + common_c2s_ipfrag_num as tcp_c2s_ip_fragments, + common_s2c_ipfrag_num as tcp_s2c_ip_fragments, + common_c2s_tcp_lostlen as tcp_c2s_lost_bytes, + common_s2c_tcp_lostlen as tcp_s2c_lost_bytes, + common_c2s_tcp_unorder_num as tcp_c2s_o3_pkts, + common_s2c_tcp_unorder_num as tcp_s2c_o3_pkts, + common_c2s_pkt_retrans as tcp_c2s_rtx_pkts, + common_s2c_pkt_retrans as tcp_s2c_rtx_pkts, + common_c2s_byte_retrans as tcp_c2s_rtx_bytes, + common_s2c_byte_retrans as tcp_s2c_rtx_bytes, + common_first_ttl as tcp_rtt_ms, + common_tcp_client_isn as tcp_client_isn, + common_tcp_server_isn as tcp_server_isn, + common_packet_capture_file as packet_capture_file, + common_in_src_mac as in_src_mac, + common_out_src_mac as out_src_mac, + common_in_dest_mac as in_dest_mac, + common_out_dest_mac as out_dest_mac, + common_tunnels as encapsulation, + common_has_dup_traffic as dup_traffic_flag, + common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, + common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc +FROM $voip_record_table_src +where common_recv_time >= $input_time_start and common_recv_time < $input_time_end +;" + # echo "sql:$voip_record_table_src => $voip_record_table_dest where common_recv_time >= $input_time_start and common_recv_time < $input_time_end" + clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql" + rst_code=$? + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + if [ $rst_code -eq 0 ]; then + insert_success_cnt=$(($insert_success_cnt+1)) + echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + else + insert_fail_cnt=$(($insert_fail_cnt+1)) + echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + fi +} + +migrate_proxy_event_table_one_part(){ + input_time_start=$1 + input_time_end=$2 + insert_sql="insert into $proxy_event_table_dest +( + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + -- security_rule_list, + -- security_action, + -- monitor_rule_list, + shaping_rule_list, + -- proxy_rule_list, + -- statistics_rule_list, + -- sc_rule_list, + -- sc_rsp_raw, + -- sc_rsp_decrypted, + -- proxy_action, + -- proxy_pinning_status, + -- proxy_intercept_status, + -- proxy_passthrough_reason, + -- proxy_client_side_latency_ms, + -- proxy_server_side_latency_ms, + -- proxy_client_side_version, + -- proxy_server_side_version, + -- proxy_cert_verify, + -- proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + -- client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + -- apn, + server_ip, + server_port, + -- server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + -- app_debug_info, + -- app_content, + fqdn_category_list, + -- ip_protocol, + decoded_path, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + -- http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + doh_url, + doh_host, + doh_request_line, + doh_response_line, + doh_cookie, + doh_referer, + doh_user_agent, + doh_content_length, + doh_content_type, + doh_set_cookie, + doh_version, + doh_message_id, + doh_qr, + doh_opcode, + doh_aa, + doh_tc, + doh_rd, + doh_ra, + doh_rcode, + doh_qdcount, + doh_ancount, + doh_nscount, + doh_arcount, + doh_qname, + doh_qtype, + doh_qclass, + doh_cname, + doh_sub, + doh_rr, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +) +SELECT + common_recv_time as recv_time, + common_log_id as log_id, + common_schema_type as decoded_as, + common_stream_trace_id as session_id, + common_start_timestamp_ms as start_timestamp_ms, + common_end_timestamp_ms as end_timestamp_ms, + common_con_duration_ms as duration_ms, + common_establish_latency_ms as tcp_handshake_latency_ms, + common_ingestion_time as ingestion_time, + common_processing_time as processing_time, + -- common_insert_time as insert_time, + common_device_id as device_id, + common_egress_link_id as out_link_id, + common_ingress_link_id as in_link_id, + common_device_tag as device_tag, + common_data_center as data_center, + common_device_group as device_group, + common_sled_ip as sled_ip, + common_address_type as address_type, + common_vsys_id as vsys_id, + common_t_vsys_id as t_vsys_id, + common_flags as flags, + common_flags_identify_info as flags_identify_info, + -- as security_rule_list, + -- as security_action, + -- as monitor_rule_list, + common_shaping_rule_ids as shaping_rule_list, + -- as proxy_rule_list, + -- as statistics_rule_list, + -- as sc_rule_list, + -- as sc_rsp_raw, + -- as sc_rsp_decrypted, + -- as proxy_action, + -- as proxy_pinning_status, + -- as proxy_intercept_status, + -- as proxy_passthrough_reason, + -- as proxy_client_side_latency_ms, + -- as proxy_server_side_latency_ms, + -- as proxy_client_side_version, + -- as proxy_server_side_version, + -- as proxy_cert_verify, + -- as proxy_intercept_error, + common_mirrored_pkts as monitor_mirrored_pkts, + common_mirrored_bytes as monitor_mirrored_bytes, + common_client_ip as client_ip, + common_client_port as client_port, + -- common_client_os_name as client_os_desc, + common_client_location as client_geolocation, + common_client_asn as client_asn, + common_subscriber_id as subscriber_id, + common_imei as imei, + common_imsi as imsi, + common_phone_number as phone_number, + -- as apn, + common_server_ip as server_ip, + common_server_port as server_port, + -- common_server_os_name as server_os_desc, + common_server_location as server_geolocation, + common_server_asn as server_asn, + common_server_fqdn as server_fqdn, + common_server_domain as server_domain, + common_app_full_path as app_transition, + common_app_label as app, + -- app_debug_info, + -- common_app_behavior as app_content, + common_service_category as fqdn_category_list, + -- ip_protocol, + common_protocol_label as decoded_path, + http_url as http_url, + http_host as http_host, + http_request_line as http_request_line, + http_response_line as http_response_line, + http_request_body as http_request_body, + http_response_body as http_response_body, + http_proxy_flag as http_proxy_flag, + http_sequence as http_sequence, + http_cookie as http_cookie, + http_referer as http_referer, + http_user_agent as http_user_agent, + http_request_content_length as http_request_content_length, + http_request_content_type as http_request_content_type, + http_response_content_length as http_response_content_length, + http_response_content_type as http_response_content_type, + http_set_cookie as http_set_cookie, + http_version as http_version, + -- http_status_code as http_status_code, + http_response_latency_ms as http_response_latency_ms, + http_session_duration_ms as http_session_duration_ms, + http_action_file_size as http_action_file_size, + doh_url as doh_url, + doh_host as doh_host, + doh_request_line as doh_request_line, + doh_response_line as doh_response_line, + doh_cookie as doh_cookie, + doh_referer as doh_referer, + doh_user_agent as doh_user_agent, + doh_content_length as doh_content_length, + doh_content_type as doh_content_type, + doh_set_cookie as doh_set_cookie, + doh_version as doh_version, + doh_message_id as doh_message_id, + doh_qr as doh_qr, + doh_opcode as doh_opcode, + doh_aa as doh_aa, + doh_tc as doh_tc, + doh_rd as doh_rd, + doh_ra as doh_ra, + doh_rcode as doh_rcode, + doh_qdcount as doh_qdcount, + doh_ancount as doh_ancount, + doh_nscount as doh_nscount, + doh_arcount as doh_arcount, + doh_qname as doh_qname, + doh_qtype as doh_qtype, + doh_qclass as doh_qclass, + doh_cname as doh_cname, + doh_sub as doh_sub, + doh_rr as doh_rr, + common_c2s_pkt_num as sent_pkts, + common_s2c_pkt_num as received_pkts, + common_c2s_byte_num as sent_bytes, + common_s2c_byte_num as received_bytes, + common_c2s_ipfrag_num as tcp_c2s_ip_fragments, + common_s2c_ipfrag_num as tcp_s2c_ip_fragments, + common_c2s_tcp_lostlen as tcp_c2s_lost_bytes, + common_s2c_tcp_lostlen as tcp_s2c_lost_bytes, + common_c2s_tcp_unorder_num as tcp_c2s_o3_pkts, + common_s2c_tcp_unorder_num as tcp_s2c_o3_pkts, + common_c2s_pkt_retrans as tcp_c2s_rtx_pkts, + common_s2c_pkt_retrans as tcp_s2c_rtx_pkts, + common_c2s_byte_retrans as tcp_c2s_rtx_bytes, + common_s2c_byte_retrans as tcp_s2c_rtx_bytes, + common_first_ttl as tcp_rtt_ms, + common_tcp_client_isn as tcp_client_isn, + common_tcp_server_isn as tcp_server_isn, + common_packet_capture_file as packet_capture_file, + common_in_src_mac as in_src_mac, + common_out_src_mac as out_src_mac, + common_in_dest_mac as in_dest_mac, + common_out_dest_mac as out_dest_mac, + common_tunnels as encapsulation, + common_has_dup_traffic as dup_traffic_flag, + common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, + common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc +FROM $proxy_event_table_src +where common_recv_time >= $input_time_start and common_recv_time < $input_time_end +;" + clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql" + rst_code=$? + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + if [ $rst_code -eq 0 ]; then + insert_success_cnt=$(($insert_success_cnt+1)) + echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + else + insert_fail_cnt=$(($insert_fail_cnt+1)) + echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + fi +} + +migrate_dos_event_table_one_part(){ + input_time_start=$1 + input_time_end=$2 + insert_sql="insert into $dos_event_table_dest +( + vsys_id, + recv_time, + log_id, + profile_id, + start_time, + end_time, + attack_type, + severity, + conditions, + destination_ip, + destination_country, + source_ip_list, + source_country_list, + session_rate, + packet_rate, + bit_rate +) +SELECT + vsys_id as vsys_id, + start_time as recv_time, + log_id as log_id, + profile_id as profile_id, + start_time as start_time, + end_time as end_time, + attack_type as attack_type, + severity as severity, + conditions as conditions, + destination_ip as destination_ip, + destination_country as destination_country, + source_ip_list as source_ip_list, + source_country_list as source_country_list, + session_rate as session_rate, + packet_rate as packet_rate, + bit_rate as bit_rate +FROM $dos_event_table_src +where start_time >= $input_time_start and start_time < $input_time_end +;" + clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql" + rst_code=$? + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + if [ $rst_code -eq 0 ]; then + insert_success_cnt=$(($insert_success_cnt+1)) + echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + else + insert_fail_cnt=$(($insert_fail_cnt+1)) + echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`" + fi +} + +slice_time_interval=$((60*slice_interval_minute)) +data_split_part=$(((timestamp_end-timestamp_start)/slice_time_interval)) + +# 输出划分的时间段 +show_time_range(){ + echo "split time ranges:" + for ((i = 0 ; i <= $data_split_part ; i++)) + do + input_time_end=$((timestamp_end-i*slice_time_interval)) + input_time_start=$((input_time_end-slice_time_interval)) + + if [ $i -eq $data_split_part ]; then + # 正好均分 + if [ $input_time_end -le $timestamp_start ]; then + continue + fi + input_time_start=$timestamp_start + fi + + + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + echo "time_range:$start_time, $end_time" + done +} + +# 迁移session_record +migrate_session_record_table(){ + echo "**********start migrate session_record table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********" + + # 遍历每个时间段数据 + for ((i = 0 ; i <= $data_split_part ; i++)) + do + input_time_end=$((timestamp_end-i*slice_time_interval)) + input_time_start=$((input_time_end-slice_time_interval)) + + if [ $i -eq $data_split_part ]; then + # 正好均分 + if [ $input_time_end -le $timestamp_start ]; then + continue + fi + input_time_start=$timestamp_start + fi + + + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + + echo "start session_record part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + # 迁移此时间段数据 + migrate_session_record_table_one_part $input_time_start $input_time_end + echo "end session_record part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + echo "" + done + + echo "**********end migrate session_record table `date --utc +'%Y-%m-%d %H:%M:%S'`**********" +} + +# 迁移security_event +migrate_security_event_table(){ + echo "**********start migrate security_event table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********" + + # 遍历每个时间段数据 + for ((i = 0 ; i <= $data_split_part ; i++)) + do + input_time_end=$((timestamp_end-i*slice_time_interval)) + input_time_start=$((input_time_end-slice_time_interval)) + + if [ $i -eq $data_split_part ]; then + # 正好均分 + if [ $input_time_end -le $timestamp_start ]; then + continue + fi + input_time_start=$timestamp_start + fi + + + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + + echo "start security_event part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + # 迁移此时间段数据 + migrate_security_event_table_one_part $input_time_start $input_time_end + echo "end security_event part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + echo "" + done + + echo "**********end migrate security_event table `date --utc +'%Y-%m-%d %H:%M:%S'`**********" +} + +# 迁移monitor_event +migrate_monitor_event_table(){ + echo "**********start migrate monitor_event table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********" + + # 遍历每个时间段数据 + for ((i = 0 ; i <= $data_split_part ; i++)) + do + input_time_end=$((timestamp_end-i*slice_time_interval)) + input_time_start=$((input_time_end-slice_time_interval)) + + if [ $i -eq $data_split_part ]; then + # 正好均分 + if [ $input_time_end -le $timestamp_start ]; then + continue + fi + input_time_start=$timestamp_start + fi + + + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + + echo "start monitor_event part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + # 迁移此时间段数据 + migrate_monitor_event_table_one_part $input_time_start $input_time_end + echo "end monitor_event part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + echo "" + done + + echo "**********end migrate monitor_event table `date --utc +'%Y-%m-%d %H:%M:%S'`**********" +} + +# 迁移transaction_record +migrate_transaction_record_table(){ + echo "**********start migrate transaction_record table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********" + + # 遍历每个时间段数据 + for ((i = 0 ; i <= $data_split_part ; i++)) + do + input_time_end=$((timestamp_end-i*slice_time_interval)) + input_time_start=$((input_time_end-slice_time_interval)) + + if [ $i -eq $data_split_part ]; then + # 正好均分 + if [ $input_time_end -le $timestamp_start ]; then + continue + fi + input_time_start=$timestamp_start + fi + + + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + + echo "start transaction_record part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + # 迁移此时间段数据 + migrate_transaction_record_table_one_part $input_time_start $input_time_end + echo "end transaction_record part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + echo "" + done + + echo "**********end migrate transaction_record table `date --utc +'%Y-%m-%d %H:%M:%S'`**********" +} + +# 迁移voip_record +migrate_voip_record_table(){ + echo "**********start migrate voip_record table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********" + + # 遍历每个时间段数据 + for ((i = 0 ; i <= $data_split_part ; i++)) + do + input_time_end=$((timestamp_end-i*slice_time_interval)) + input_time_start=$((input_time_end-slice_time_interval)) + + if [ $i -eq $data_split_part ]; then + # 正好均分 + if [ $input_time_end -le $timestamp_start ]; then + continue + fi + input_time_start=$timestamp_start + fi + + + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + + echo "start voip_record part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + # 迁移此时间段数据 + migrate_voip_record_table_one_part $input_time_start $input_time_end + echo "end voip_record part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + echo "" + done + + echo "**********end migrate voip_record table `date --utc +'%Y-%m-%d %H:%M:%S'`**********" +} + +# 迁移proxy_event +migrate_proxy_event_table(){ + echo "**********start migrate $table table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********" + + # 遍历每个时间段数据 + for ((i = 0 ; i <= $data_split_part ; i++)) + do + input_time_end=$((timestamp_end-i*slice_time_interval)) + input_time_start=$((input_time_end-slice_time_interval)) + + if [ $i -eq $data_split_part ]; then + # 正好均分 + if [ $input_time_end -le $timestamp_start ]; then + continue + fi + input_time_start=$timestamp_start + fi + + + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + + echo "start $table part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + # 迁移此时间段数据 + migrate_proxy_event_table_one_part $input_time_start $input_time_end + echo "end $table part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + echo "" + done + + echo "**********end migrate $table table `date --utc +'%Y-%m-%d %H:%M:%S'`**********" +} + +# 迁移dos_event +migrate_dos_event_table(){ + echo "**********start migrate $table table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********" + + # 遍历每个时间段数据 + for ((i = 0 ; i <= $data_split_part ; i++)) + do + input_time_end=$((timestamp_end-i*slice_time_interval)) + input_time_start=$((input_time_end-slice_time_interval)) + + if [ $i -eq $data_split_part ]; then + # 正好均分 + if [ $input_time_end -le $timestamp_start ]; then + continue + fi + input_time_start=$timestamp_start + fi + + + start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + + echo "start $table part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + # 迁移此时间段数据 + migrate_dos_event_table_one_part $input_time_start $input_time_end + echo "end $table part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`" + echo "" + done + + echo "**********end migrate $table table `date --utc +'%Y-%m-%d %H:%M:%S'`**********" +} + +# 输出划分的时间段 +show_time_range + +# 校验迁移表参数 +case $table in +"session_record") + # 迁移session_record + migrate_session_record_table +;; +"security_event") + # 迁移security_event + migrate_security_event_table +;; +"monitor_event") + # 迁移monitor_event + migrate_monitor_event_table +;; +"transaction_record") + # 迁移transaction_record + migrate_transaction_record_table +;; +"voip_record") + # 迁移voip_record + migrate_voip_record_table +;; +"proxy_event") + # 迁移proxy_event + migrate_proxy_event_table +;; +"dos_event") + # 迁移dos_event + migrate_dos_event_table +;; +*) + echo "所迁移表${table}不在范围:session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event" +;; +esac + +echo "migrate_table_end:`date --utc +'%Y-%m-%d %H:%M:%S'`, 成功迁移数据段:${insert_success_cnt}, 失败迁移数据段:${insert_fail_cnt}" + +echo "成功迁移数据段:${insert_success_cnt}, 失败迁移数据段:${insert_fail_cnt}" + +echo "#####################end `date --utc +'%Y-%m-%d %H:%M:%S'`#######################" \ No newline at end of file diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/migrate_table_2402/start_migrate_local_table_to_2402.sh b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/migrate_table_2402/start_migrate_local_table_to_2402.sh new file mode 100644 index 0000000..74786f9 --- /dev/null +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/migrate_table_2402/start_migrate_local_table_to_2402.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# 本脚本逐时间段按最新往前顺序迁移clickhouse数据，TSG24.01日志重组后数据迁移 + +# 迁移表 参数，可选值：session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event +table=$1 +# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00" +data_start_time=$2 +# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00" +data_end_time=$3 +# 每批迁移数据时间段长度(分钟) 参数, 例如:240 +slice_interval_minute=$4 + +timestamp_start=`date --utc --date="$data_start_time" +%s` +timestamp_end=`date --utc --date="$data_end_time" +%s` + +# 校验迁移表参数 +case $table in +"session_record") + # 迁移session_record + echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute" +;; +"security_event") + # 迁移security_event + echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute" +;; +"monitor_event") + # 迁移monitor_event + echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute" +;; +"transaction_record") + # 迁移transaction_record + echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute" +;; +"voip_record") + # 迁移voip_record + echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute" +;; +"proxy_event") + # 迁移proxy_event + echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute" +;; +"dos_event") + # 迁移dos_event + echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute" +;; +*) + echo "所迁移表${table}不在范围:session_record，security_event，monitor_event，transaction_record，voip_record，proxy_event，dos_event" + exit 1 +;; +esac + +# 校验时间参数 +if [ -z "$timestamp_start" ]; then + echo "data_start_time fmt err" + exit 1 +fi + +if [ -z "$timestamp_end" ]; then + echo "data_end_time fmt err" + exit 1 +fi + +if [ $timestamp_start -ge $timestamp_end ]; then + echo "date range err" + exit 1 +fi + +if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then + echo "slice_interval_minute参数必须是正确的分钟数" + exit 1 +fi + +# 后台执行，输出日志 +nohup ./migrate_local_table_to_2402.sh "$table" "$data_start_time" "$data_end_time" $slice_interval_minute > "log_$table.txt" 2>&1 & +echo "已启动迁移${table}表任务，时间范围[$data_start_time, $data_end_time], 每批迁移段分钟:$slice_interval_minute, 日志输出到:log_$table.txt。请查看日志文件确认每段数据迁移情况" + diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_2307_to_tmp_2307.sql b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_2307_to_tmp_2307.sql new file mode 100644 index 0000000..62fdbde --- /dev/null +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_2307_to_tmp_2307.sql @@ -0,0 +1,46 @@ +set distributed_ddl_task_timeout = 180; + +-- 删除源表同步子表物化视图 + + +-- 源表rename到历史表 +RENAME TABLE tsg_galaxy_v3.session_record_local to tsg_galaxy_tmp.session_record_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.security_event_local to tsg_galaxy_tmp.security_event_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.transaction_record_local to tsg_galaxy_tmp.transaction_record_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.voip_record_local to tsg_galaxy_tmp.voip_record_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.proxy_event_local to tsg_galaxy_tmp.proxy_event_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.dos_event_local to tsg_galaxy_tmp.dos_event_local_old on cluster ck_cluster; + + +-- 创建源分布式表old +create table IF NOT EXISTS tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local_old,rand()); + +create table IF NOT EXISTS tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local_old,rand()); + +create table IF NOT EXISTS tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query( + log_id UInt64, + profile_id UInt64, + start_time Int64 +) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local_old,rand()); + diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md new file mode 100644 index 0000000..f05d893 --- /dev/null +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md @@ -0,0 +1,307 @@ +由于各环境当前使用tsg版本与升级的版本均不同，故在此提供通用步骤，因主键与字段不同，旧版本统一升级至23.07版本进行处理 +具体步骤： + +Step1 ：停止入库任务。 +Step2 ：旧版本clickhouse库表升级到23.07版本 +Step3 ：tsg_galaxy_tmp数据库新建24.02版本库表,修改建表语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp +Step4 ：23.07版本库表迁移至tsg_galaxy_tmp数据库 +Step5 : tsg_galaxy_v3数据库新建目标版本库表（如24.04） +Step6 : 启动入库任务->tsg_galaxy_v3 +Step7 : 迁移脚本迁移tsg_galaxy_tmp(23.07)->tsg_galaxy_tmp(24.02) +Step8 : 升级tsg_galaxy_tmp(24.02)->升级tsg_galaxy_tmp目标版本如（24.04） +Step9 : 命令迁移tsg_galaxy_tmp目标版本如（24.04）->tsg_galaxy_v3目标版本如（24.04）,按照（天）partition手动迁移 + +说明 + +请按步骤依次执行，执行脚本报错时联系研发处理后再执行之后的步骤。 +所有ck步骤都需要在query节点执行 +执行所有sql语句之前需要停止日志留存调度任务，确保ck中无分布式ddl语句H执行，否则执行的sql会阻塞住，影响后续步骤执行 +验证sql需要在query节点执行 +clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1" +若返回结果为空则可执行升级步骤，否则需要等待。 + + +一、停止旧表ck入库任务 +停止旧表ck入库任务 + +二、旧版本clickhouse库表升级到23.07版本,依次执行版本升级语句 + + +三、临时库初始化24.02版本库表 + +1.执行2402版本初始化建表语句 + + +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < init_tsg_galaxy_tmp_24_02_table.sql + + + +2.校验表结构 + + +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < check_tsg_galaxy_tmp_24_02_table.sql + + +无报错信息说明校验通过 + +四、23.07版本库表迁移至tsg_galaxy_tmp数据库 + +1.迁移sql +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_v3_2307_to_tmp_2307.sql + + +2.查看tmp库old表数据量 + +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_tmp_old_table_row_count.sql + +五、tsg_galaxy_v3数据库新建目标版本库表 + +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 对应版本初始化sql + +六、启动ck入库任务 + +1.启动目标版本ck入库任务 + + +七、离线脚本同步历史数据至临时数据库2402版本库表 +在query节点执行以下步骤，iplist.txt中为ck所有data节点ip地址。 +步骤描述： + +1.进入migrate_table_2402文件夹,使脚本可执行 + + +chmod +x ./*.sh + + + +2.分发迁移脚本到data节点 + + +./01_send_migrate_table_scripts.sh + + + +2.选择迁移某个表，同步需要时间区间的数据，时间区间:[实时同步任务开始时间向前推n天, 实时同步任务开始时间)，时间区间为左闭右开，不包含结束时间点。 + + +# 迁移security_event表 +./02_start_migrate_table.sh security_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60 + + + +3.监控data节点迁移情况，所有表迁移完成后，确认每个节点同步数据成功/失败批次数，如有失败批次确认是否需要处理 + + +# 监控security_event表迁移 +./03_monitor_migrate_table.sh security_event + + + +4.选择下个张需要迁移的表，重复2-4步骤。支持选择迁移的表有: security_event, monitor_event, session_record, transaction_record, voip_record, proxy_event, dos_event。 + +迁移和监控各个表执行命令示例： + +# 迁移security_event表 +./02_start_migrate_table.sh security_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60 +# 监控security_event表迁移 +./03_monitor_migrate_table.sh security_event + + +# 迁移monitor_event表 +./02_start_migrate_table.sh monitor_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60 +# 监控monitor_event表迁移 +./03_monitor_migrate_table.sh monitor_event + + +# 迁移session_record表 +./02_start_migrate_table.sh session_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60 +# 监控session_record表迁移 +./03_monitor_migrate_table.sh session_record + + +# 迁移transaction_record表 +./02_start_migrate_table.sh transaction_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60 +# 监控transaction_record表迁移 +./03_monitor_migrate_table.sh transaction_record + + +# 迁移voip_record表 +./02_start_migrate_table.sh voip_record "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60 +# 监控voip_record表迁移 +./03_monitor_migrate_table.sh voip_record + + +# 迁移proxy_event表 +./02_start_migrate_table.sh proxy_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60 +# 监控proxy_event表迁移 +./03_monitor_migrate_table.sh proxy_event + + +# 迁移dos_event表 +./02_start_migrate_table.sh dos_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60 +# 监控dos_event表迁移 +./03_monitor_migrate_table.sh dos_event + + +迁移日志无报错，数据迁移完成。 +如果有数据迁移失败批次，查看新老表迁移数据量对应情况(ck每台data节点)： + +-- security_event + +SELECT + date_trunc('day', toDateTime(common_recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.security_event_local_old +WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00') + and common_action in (16, 96) +group by date_trunc('day', toDateTime(common_recv_time)) +order by d +; + +SELECT + date_trunc('day', toDateTime(recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.security_event_local +WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(recv_time)) +order by d +; + +-- monitor_event + +SELECT + date_trunc('day', toDateTime(common_recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.security_event_local_old +WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00') + and common_action = 1 +group by date_trunc('day', toDateTime(common_recv_time)) +order by d +; + +SELECT + date_trunc('day', toDateTime(recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.monitor_event_local +WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(recv_time)) +order by d +; + +-- session_record + +SELECT + date_trunc('day', toDateTime(common_recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.session_record_local_old +WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(common_recv_time)) +order by d +; + +SELECT + date_trunc('day', toDateTime(recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.session_record_local +WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(recv_time)) +order by d +; + +-- transaction_record + +SELECT + date_trunc('day', toDateTime(common_recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.transaction_record_local_old +WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(common_recv_time)) +order by d +; + +SELECT + date_trunc('day', toDateTime(recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.transaction_record_local +WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(recv_time)) +order by d +; + +-- voip_record + +SELECT + date_trunc('day', toDateTime(common_recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.voip_record_local_old +WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(common_recv_time)) +order by d +; + +SELECT + date_trunc('day', toDateTime(recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.voip_record_local +WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(recv_time)) +order by d +; + +-- proxy_event + +SELECT + date_trunc('day', toDateTime(common_recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.proxy_event_local_old +WHERE common_recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and common_recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(common_recv_time)) +order by d +; + +SELECT + date_trunc('day', toDateTime(recv_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.proxy_event_local +WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(recv_time)) +order by d +; + +-- dos_event + +SELECT + date_trunc('day', toDateTime(start_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.dos_event_local_old +WHERE start_time>= toUnixTimestamp('2024-01-10 00:00:00') and start_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(start_time)) +order by d +; + +SELECT + date_trunc('day', toDateTime(start_time)) d, + COUNT(1) cnt +FROM tsg_galaxy_tmp.dos_event_local +WHERE start_time >= toUnixTimestamp('2024-01-10 00:00:00') and start_time < toUnixTimestamp('2024-01-20 00:00:00') +group by date_trunc('day', toDateTime(start_time)) +order by d +; + + +八、升级tsg_galaxy_tmp(24.02)->升级tsg_galaxy_tmp目标版本如（24.04）主要目的使临时库与实际库表结构相同 + +从24.03开始依次执行直至目标版本（注意所有版本升级sql需要将tsg_galaxy_v3替换tsg_galaxy_tmp,包括校验sql） + + +九、命令迁移tsg_galaxy_tmp目标版本如（24.04）->tsg_galaxy_v3目标版本如（24.04）,按照（天）partition手动迁移 + +登录命令行之后手动按需执行需要迁移的分区（如下示例迁移2024年4月1日的数据） +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** + +ALTER TABLE tsg_galaxy_tmp.session_record_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.session_record_local; +ALTER TABLE tsg_galaxy_tmp.security_event_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.security_event_local; +ALTER TABLE tsg_galaxy_tmp.transaction_record_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.transaction_record_local; +ALTER TABLE tsg_galaxy_tmp.voip_record_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.voip_record_local; +ALTER TABLE tsg_galaxy_tmp.proxy_event_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.proxy_event_local; +ALTER TABLE tsg_galaxy_tmp.dos_event_local on cluster ck_cluster move partition 20240401 to tsg_galaxy_v3.dos_event_local;