添加数据迁移脚本
This commit is contained in:
houjinchuan
1
Clickhouse最新全量建表语句/数据迁移脚本/iplist.txt
Normal file
1
Clickhouse最新全量建表语句/数据迁移脚本/iplist.txt
Normal file
@@ -0,0 +1 @@
|
||||
192.168.41.30
|
||||
339
Clickhouse最新全量建表语句/数据迁移脚本/mig.sh
Normal file
339
Clickhouse最新全量建表语句/数据迁移脚本/mig.sh
Normal file
@@ -0,0 +1,339 @@
|
||||
#!/bin/bash
|
||||
|
||||
#本脚本安装逐台服务器逐时间段的顺序迁移clickhouse数据
|
||||
data_start_time="2021-09-10 00:00:00" #迁移数据common_recv_time 起始时间
|
||||
data_end_time="2021-09-15 23:59:59"
|
||||
query_timeout=36000 # 执行insert 的超时时间秒
|
||||
data_split_part=170 #切割次数 每段时间区间= (data_start_time-data_end_time)/data_split_part
|
||||
ck_username="default" #ck用户名
|
||||
ck_password="galaxy2019" #ck密码
|
||||
data_destination_table="tsg_galaxy_v3.session_record_local_t5" #目的表名
|
||||
data_source_table="tsg_galaxy_xj.connection_record_log_local" #源表名
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
timestamp_start=`date --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --date="$data_end_time" +%s`
|
||||
|
||||
for ip in `cat iplist.txt`
|
||||
do echo " $ip start"
|
||||
slice_time_interval=$(((timestamp_end-timestamp_start)/data_split_part))
|
||||
if [ $timestamp_start -lt $timestamp_end ];then
|
||||
for ((i = 0 ; i < $data_split_part ; i++))
|
||||
do
|
||||
input_time_start=$((timestamp_start+i*slice_time_interval))
|
||||
input_time_end=$((input_time_start+slice_time_interval))
|
||||
clickhouse-client -h $ip --port 9001 -m -u $ck_username --password $ck_password --max_final_threads=1 --max_insert_threads=1 --max_threads=1 --max_execution_time=$query_timeout --query="insert
|
||||
into
|
||||
$data_destination_table
|
||||
(common_log_id ,
|
||||
common_service ,
|
||||
common_recv_time ,
|
||||
common_direction ,
|
||||
common_l4_protocol ,
|
||||
common_address_type ,
|
||||
common_schema_type ,
|
||||
common_policy_id ,
|
||||
common_user_tags ,
|
||||
common_action ,
|
||||
common_sub_action ,
|
||||
common_user_region ,
|
||||
common_client_ip ,
|
||||
common_client_port ,
|
||||
common_internal_ip ,
|
||||
common_entrance_id ,
|
||||
common_device_id ,
|
||||
common_isp ,
|
||||
common_device_tag ,
|
||||
common_data_center ,
|
||||
common_encapsulation ,
|
||||
common_sled_ip ,
|
||||
common_client_location ,
|
||||
common_client_asn ,
|
||||
common_subscriber_id ,
|
||||
common_server_ip ,
|
||||
common_server_port ,
|
||||
common_external_ip ,
|
||||
common_server_location ,
|
||||
common_server_asn ,
|
||||
common_protocol_label ,
|
||||
common_app_label ,
|
||||
common_l7_protocol ,
|
||||
common_sessions ,
|
||||
common_c2s_pkt_num ,
|
||||
common_s2c_pkt_num ,
|
||||
common_c2s_byte_num ,
|
||||
common_s2c_byte_num ,
|
||||
common_start_time ,
|
||||
common_end_time ,
|
||||
common_establish_latency_ms ,
|
||||
common_con_duration_ms ,
|
||||
common_stream_dir ,
|
||||
common_address_list ,
|
||||
common_has_dup_traffic ,
|
||||
common_stream_error ,
|
||||
common_stream_trace_id ,
|
||||
common_link_info_c2s ,
|
||||
common_link_info_s2c ,
|
||||
common_c2s_ipfrag_num ,
|
||||
common_s2c_ipfrag_num ,
|
||||
common_c2s_tcp_lostlen ,
|
||||
common_s2c_tcp_lostlen ,
|
||||
common_c2s_tcp_unorder_num ,
|
||||
common_s2c_tcp_unorder_num ,
|
||||
common_first_ttl ,
|
||||
common_processing_time ,
|
||||
http_url ,
|
||||
http_host ,
|
||||
http_domain ,
|
||||
http_request_line ,
|
||||
http_response_line ,
|
||||
http_request_header ,
|
||||
http_response_header ,
|
||||
http_request_body ,
|
||||
http_response_body ,
|
||||
http_request_body_key ,
|
||||
http_response_body_key ,
|
||||
http_proxy_flag ,
|
||||
http_sequence ,
|
||||
http_snapshot ,
|
||||
http_cookie ,
|
||||
http_referer ,
|
||||
http_user_agent ,
|
||||
http_content_length ,
|
||||
http_content_type ,
|
||||
http_set_cookie ,
|
||||
http_version ,
|
||||
http_session_duration_ms ,
|
||||
http_action_file_size ,
|
||||
mail_protocol_type ,
|
||||
mail_account ,
|
||||
mail_to_cmd ,
|
||||
mail_from_cmd ,
|
||||
mail_from ,
|
||||
mail_to ,
|
||||
mail_cc ,
|
||||
mail_bcc ,
|
||||
mail_subject ,
|
||||
mail_subject_charset ,
|
||||
mail_content ,
|
||||
mail_content_charset ,
|
||||
mail_attachment_name ,
|
||||
mail_attachment_name_charset ,
|
||||
mail_attachment_content ,
|
||||
mail_eml_file ,
|
||||
mail_snapshot ,
|
||||
dns_message_id ,
|
||||
dns_qr ,
|
||||
dns_opcode ,
|
||||
dns_aa ,
|
||||
dns_tc ,
|
||||
dns_rd ,
|
||||
dns_ra ,
|
||||
dns_rcode ,
|
||||
dns_qdcount ,
|
||||
dns_ancount ,
|
||||
dns_nscount ,
|
||||
dns_arcount ,
|
||||
dns_qname ,
|
||||
dns_qtype ,
|
||||
dns_qclass ,
|
||||
dns_cname ,
|
||||
dns_sub ,
|
||||
dns_rr ,
|
||||
ssl_version ,
|
||||
ssl_sni ,
|
||||
ssl_san ,
|
||||
ssl_cn ,
|
||||
ssl_pinningst ,
|
||||
ssl_intercept_state ,
|
||||
ssl_server_side_latency ,
|
||||
ssl_client_side_latency ,
|
||||
ssl_server_side_version ,
|
||||
ssl_client_side_version ,
|
||||
ssl_cert_verify ,
|
||||
ssl_error ,
|
||||
ssl_con_latency_ms ,
|
||||
ssl_ja3_fingerprint ,
|
||||
ssl_ja3_hash ,
|
||||
quic_version ,
|
||||
quic_sni ,
|
||||
quic_user_agent ,
|
||||
ftp_account ,
|
||||
ftp_url ,
|
||||
ftp_content ,
|
||||
bgp_type ,
|
||||
bgp_as_num ,
|
||||
bgp_route ,
|
||||
voip_calling_account ,
|
||||
voip_called_account ,
|
||||
voip_calling_number ,
|
||||
voip_called_number ,
|
||||
streaming_media_url ,
|
||||
streaming_media_protocol ,
|
||||
app_extra_info, common_vsys_id)
|
||||
select
|
||||
common_log_id,
|
||||
common_service ,
|
||||
common_recv_time ,
|
||||
common_direction ,
|
||||
common_l4_protocol ,
|
||||
common_address_type ,
|
||||
common_schema_type ,
|
||||
common_policy_id ,
|
||||
common_user_tags ,
|
||||
common_action ,
|
||||
common_sub_action ,
|
||||
common_user_region ,
|
||||
common_client_ip ,
|
||||
common_client_port ,
|
||||
common_internal_ip ,
|
||||
common_entrance_id ,
|
||||
common_device_id ,
|
||||
common_isp ,
|
||||
common_device_tag ,
|
||||
common_data_center ,
|
||||
common_encapsulation ,
|
||||
common_sled_ip ,
|
||||
common_client_location ,
|
||||
common_client_asn ,
|
||||
common_subscriber_id ,
|
||||
common_server_ip ,
|
||||
common_server_port ,
|
||||
common_external_ip ,
|
||||
common_server_location ,
|
||||
common_server_asn ,
|
||||
common_protocol_label ,
|
||||
common_app_label ,
|
||||
common_l7_protocol ,
|
||||
common_sessions ,
|
||||
common_c2s_pkt_num ,
|
||||
common_s2c_pkt_num ,
|
||||
common_c2s_byte_num ,
|
||||
common_s2c_byte_num ,
|
||||
common_start_time ,
|
||||
common_end_time ,
|
||||
common_establish_latency_ms ,
|
||||
common_con_duration_ms ,
|
||||
common_stream_dir ,
|
||||
common_address_list ,
|
||||
common_has_dup_traffic ,
|
||||
common_stream_error ,
|
||||
common_log_id ,
|
||||
common_link_info_c2s ,
|
||||
common_link_info_s2c ,
|
||||
common_c2s_ipfrag_num ,
|
||||
common_s2c_ipfrag_num ,
|
||||
common_c2s_tcp_lostlen ,
|
||||
common_s2c_tcp_lostlen ,
|
||||
common_c2s_tcp_unorder_num ,
|
||||
common_s2c_tcp_unorder_num ,
|
||||
common_first_ttl ,
|
||||
common_processing_time ,
|
||||
http_url ,
|
||||
http_host ,
|
||||
http_domain ,
|
||||
http_request_line ,
|
||||
http_response_line ,
|
||||
http_request_header ,
|
||||
http_response_header ,
|
||||
http_request_body ,
|
||||
http_response_body ,
|
||||
http_request_body_key ,
|
||||
http_response_body_key ,
|
||||
http_proxy_flag ,
|
||||
http_sequence ,
|
||||
http_snapshot ,
|
||||
http_cookie ,
|
||||
http_referer ,
|
||||
http_user_agent ,
|
||||
http_content_length ,
|
||||
http_content_type ,
|
||||
http_set_cookie ,
|
||||
http_version ,
|
||||
http_session_duration_ms ,
|
||||
http_action_file_size ,
|
||||
mail_protocol_type ,
|
||||
mail_account ,
|
||||
mail_to_cmd ,
|
||||
mail_from_cmd ,
|
||||
mail_from ,
|
||||
mail_to ,
|
||||
mail_cc ,
|
||||
mail_bcc ,
|
||||
mail_subject ,
|
||||
mail_subject_charset ,
|
||||
mail_content ,
|
||||
mail_content_charset ,
|
||||
mail_attachment_name ,
|
||||
mail_attachment_name_charset ,
|
||||
mail_attachment_content ,
|
||||
mail_eml_file ,
|
||||
mail_snapshot ,
|
||||
dns_message_id ,
|
||||
dns_qr ,
|
||||
dns_opcode ,
|
||||
dns_aa ,
|
||||
dns_tc ,
|
||||
dns_rd ,
|
||||
dns_ra ,
|
||||
dns_rcode ,
|
||||
dns_qdcount ,
|
||||
dns_ancount ,
|
||||
dns_nscount ,
|
||||
dns_arcount ,
|
||||
dns_qname ,
|
||||
dns_qtype ,
|
||||
dns_qclass ,
|
||||
dns_cname ,
|
||||
dns_sub ,
|
||||
dns_rr ,
|
||||
ssl_version ,
|
||||
ssl_sni ,
|
||||
ssl_san ,
|
||||
ssl_cn ,
|
||||
ssl_pinningst ,
|
||||
ssl_intercept_state ,
|
||||
ssl_server_side_latency ,
|
||||
ssl_client_side_latency ,
|
||||
ssl_server_side_version ,
|
||||
ssl_client_side_version ,
|
||||
ssl_cert_verify ,
|
||||
ssl_error ,
|
||||
ssl_con_latency_ms ,
|
||||
ssl_ja3_fingerprint ,
|
||||
ssl_ja3_hash ,
|
||||
quic_version ,
|
||||
quic_sni ,
|
||||
quic_user_agent ,
|
||||
ftp_account ,
|
||||
ftp_url ,
|
||||
ftp_content ,
|
||||
bgp_type ,
|
||||
bgp_as_num ,
|
||||
bgp_route ,
|
||||
voip_calling_account ,
|
||||
voip_called_account ,
|
||||
voip_calling_number ,
|
||||
voip_called_number ,
|
||||
streaming_media_url ,
|
||||
streaming_media_protocol ,
|
||||
app_extra_info,rand()%100 as common_vsys_id
|
||||
from
|
||||
$data_source_table
|
||||
where
|
||||
common_recv_time >= $input_time_start
|
||||
and common_recv_time < $input_time_end"
|
||||
start_time=`date -d @$input_time_start "+%Y-%m-%d %H:%M:%S"`
|
||||
end_time=`date -d @$input_time_end "+%Y-%m-%d %H:%M:%S"`
|
||||
echo " $data_destination_table common_recv_time>=$start_time and common_recv_time <=$end_time finished"
|
||||
done
|
||||
else
|
||||
echo " Error timestamp"
|
||||
continue
|
||||
fi
|
||||
echo " $ip done"
|
||||
done
|
||||
echo " Migrate Data Finished"
|
||||
Reference in New Issue
Block a user