diff --git a/Clickhouse最新全量建表语句/数据迁移脚本/iplist.txt b/Clickhouse最新全量建表语句/数据迁移脚本/iplist.txt new file mode 100644 index 0000000..64aae45 --- /dev/null +++ b/Clickhouse最新全量建表语句/数据迁移脚本/iplist.txt @@ -0,0 +1 @@ +192.168.41.30 diff --git a/Clickhouse最新全量建表语句/数据迁移脚本/mig.sh b/Clickhouse最新全量建表语句/数据迁移脚本/mig.sh new file mode 100644 index 0000000..033c744 --- /dev/null +++ b/Clickhouse最新全量建表语句/数据迁移脚本/mig.sh @@ -0,0 +1,339 @@ +#!/bin/bash + +#本脚本安装逐台服务器逐时间段的顺序迁移clickhouse数据 +data_start_time="2021-09-10 00:00:00" #迁移数据common_recv_time 起始时间 +data_end_time="2021-09-15 23:59:59" +query_timeout=36000 # 执行insert 的超时时间秒 +data_split_part=170 #切割次数 每段时间区间= (data_start_time-data_end_time)/data_split_part +ck_username="default" #ck用户名 +ck_password="galaxy2019" #ck密码 +data_destination_table="tsg_galaxy_v3.session_record_local_t5" #目的表名 +data_source_table="tsg_galaxy_xj.connection_record_log_local" #源表名 + + + + + +timestamp_start=`date --date="$data_start_time" +%s` +timestamp_end=`date --date="$data_end_time" +%s` + +for ip in `cat iplist.txt` +do echo " $ip start" + slice_time_interval=$(((timestamp_end-timestamp_start)/data_split_part)) + if [ $timestamp_start -lt $timestamp_end ];then + for ((i = 0 ; i < $data_split_part ; i++)) + do + input_time_start=$((timestamp_start+i*slice_time_interval)) + input_time_end=$((input_time_start+slice_time_interval)) + clickhouse-client -h $ip --port 9001 -m -u $ck_username --password $ck_password --max_final_threads=1 --max_insert_threads=1 --max_threads=1 --max_execution_time=$query_timeout --query="insert + into + $data_destination_table + (common_log_id , + common_service , + common_recv_time , + common_direction , + common_l4_protocol , + common_address_type , + common_schema_type , + common_policy_id , + common_user_tags , + common_action , + common_sub_action , + common_user_region , + common_client_ip , + common_client_port , + common_internal_ip , + common_entrance_id , + common_device_id , + common_isp , + common_device_tag , + common_data_center , + common_encapsulation , + common_sled_ip , + common_client_location , + common_client_asn , + common_subscriber_id , + common_server_ip , + common_server_port , + common_external_ip , + common_server_location , + common_server_asn , + common_protocol_label , + common_app_label , + common_l7_protocol , + common_sessions , + common_c2s_pkt_num , + common_s2c_pkt_num , + common_c2s_byte_num , + common_s2c_byte_num , + common_start_time , + common_end_time , + common_establish_latency_ms , + common_con_duration_ms , + common_stream_dir , + common_address_list , + common_has_dup_traffic , + common_stream_error , + common_stream_trace_id , + common_link_info_c2s , + common_link_info_s2c , + common_c2s_ipfrag_num , + common_s2c_ipfrag_num , + common_c2s_tcp_lostlen , + common_s2c_tcp_lostlen , + common_c2s_tcp_unorder_num , + common_s2c_tcp_unorder_num , + common_first_ttl , + common_processing_time , + http_url , + http_host , + http_domain , + http_request_line , + http_response_line , + http_request_header , + http_response_header , + http_request_body , + http_response_body , + http_request_body_key , + http_response_body_key , + http_proxy_flag , + http_sequence , + http_snapshot , + http_cookie , + http_referer , + http_user_agent , + http_content_length , + http_content_type , + http_set_cookie , + http_version , + http_session_duration_ms , + http_action_file_size , + mail_protocol_type , + mail_account , + mail_to_cmd , + mail_from_cmd , + mail_from , + mail_to , + mail_cc , + mail_bcc , + mail_subject , + mail_subject_charset , + mail_content , + mail_content_charset , + mail_attachment_name , + mail_attachment_name_charset , + mail_attachment_content , + mail_eml_file , + mail_snapshot , + dns_message_id , + dns_qr , + dns_opcode , + dns_aa , + dns_tc , + dns_rd , + dns_ra , + dns_rcode , + dns_qdcount , + dns_ancount , + dns_nscount , + dns_arcount , + dns_qname , + dns_qtype , + dns_qclass , + dns_cname , + dns_sub , + dns_rr , + ssl_version , + ssl_sni , + ssl_san , + ssl_cn , + ssl_pinningst , + ssl_intercept_state , + ssl_server_side_latency , + ssl_client_side_latency , + ssl_server_side_version , + ssl_client_side_version , + ssl_cert_verify , + ssl_error , + ssl_con_latency_ms , + ssl_ja3_fingerprint , + ssl_ja3_hash , + quic_version , + quic_sni , + quic_user_agent , + ftp_account , + ftp_url , + ftp_content , + bgp_type , + bgp_as_num , + bgp_route , + voip_calling_account , + voip_called_account , + voip_calling_number , + voip_called_number , + streaming_media_url , + streaming_media_protocol , + app_extra_info, common_vsys_id) +select + common_log_id, + common_service , + common_recv_time , + common_direction , + common_l4_protocol , + common_address_type , + common_schema_type , + common_policy_id , + common_user_tags , + common_action , + common_sub_action , + common_user_region , + common_client_ip , + common_client_port , + common_internal_ip , + common_entrance_id , + common_device_id , + common_isp , + common_device_tag , + common_data_center , + common_encapsulation , + common_sled_ip , + common_client_location , + common_client_asn , + common_subscriber_id , + common_server_ip , + common_server_port , + common_external_ip , + common_server_location , + common_server_asn , + common_protocol_label , + common_app_label , + common_l7_protocol , + common_sessions , + common_c2s_pkt_num , + common_s2c_pkt_num , + common_c2s_byte_num , + common_s2c_byte_num , + common_start_time , + common_end_time , + common_establish_latency_ms , + common_con_duration_ms , + common_stream_dir , + common_address_list , + common_has_dup_traffic , + common_stream_error , + common_log_id , + common_link_info_c2s , + common_link_info_s2c , + common_c2s_ipfrag_num , + common_s2c_ipfrag_num , + common_c2s_tcp_lostlen , + common_s2c_tcp_lostlen , + common_c2s_tcp_unorder_num , + common_s2c_tcp_unorder_num , + common_first_ttl , + common_processing_time , + http_url , + http_host , + http_domain , + http_request_line , + http_response_line , + http_request_header , + http_response_header , + http_request_body , + http_response_body , + http_request_body_key , + http_response_body_key , + http_proxy_flag , + http_sequence , + http_snapshot , + http_cookie , + http_referer , + http_user_agent , + http_content_length , + http_content_type , + http_set_cookie , + http_version , + http_session_duration_ms , + http_action_file_size , + mail_protocol_type , + mail_account , + mail_to_cmd , + mail_from_cmd , + mail_from , + mail_to , + mail_cc , + mail_bcc , + mail_subject , + mail_subject_charset , + mail_content , + mail_content_charset , + mail_attachment_name , + mail_attachment_name_charset , + mail_attachment_content , + mail_eml_file , + mail_snapshot , + dns_message_id , + dns_qr , + dns_opcode , + dns_aa , + dns_tc , + dns_rd , + dns_ra , + dns_rcode , + dns_qdcount , + dns_ancount , + dns_nscount , + dns_arcount , + dns_qname , + dns_qtype , + dns_qclass , + dns_cname , + dns_sub , + dns_rr , + ssl_version , + ssl_sni , + ssl_san , + ssl_cn , + ssl_pinningst , + ssl_intercept_state , + ssl_server_side_latency , + ssl_client_side_latency , + ssl_server_side_version , + ssl_client_side_version , + ssl_cert_verify , + ssl_error , + ssl_con_latency_ms , + ssl_ja3_fingerprint , + ssl_ja3_hash , + quic_version , + quic_sni , + quic_user_agent , + ftp_account , + ftp_url , + ftp_content , + bgp_type , + bgp_as_num , + bgp_route , + voip_calling_account , + voip_called_account , + voip_calling_number , + voip_called_number , + streaming_media_url , + streaming_media_protocol , + app_extra_info,rand()%100 as common_vsys_id +from + $data_source_table +where + common_recv_time >= $input_time_start + and common_recv_time < $input_time_end" + start_time=`date -d @$input_time_start "+%Y-%m-%d %H:%M:%S"` + end_time=`date -d @$input_time_end "+%Y-%m-%d %H:%M:%S"` + echo " $data_destination_table common_recv_time>=$start_time and common_recv_time <=$end_time finished" + done + else + echo " Error timestamp" + continue + fi + echo " $ip done" +done +echo " Migrate Data Finished"