diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/cat_v3_old_table_row_count.sql b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/cat_v3_old_table_row_count.sql new file mode 100644 index 0000000..e7cd556 --- /dev/null +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/cat_v3_old_table_row_count.sql @@ -0,0 +1,11 @@ +select 'session_record' as table_name, count(*) as cnt from tsg_galaxy_v3.session_record; + +select 'security_event' as table_name, count(*) as cnt from tsg_galaxy_v3.security_event; + +select 'transaction_record' as table_name, count(*) as cnt from tsg_galaxy_v3.transaction_record; + +select 'voip_record' as table_name, count(*) as cnt from tsg_galaxy_v3.voip_record; + +select 'proxy_event' as table_name, count(*) as cnt from tsg_galaxy_v3.proxy_event; + +select 'dos_event' as table_name, count(*) as cnt from tsg_galaxy_v3.dos_event; \ No newline at end of file diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/migrate_table_2402/migrate_local_table_to_2402.sh b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/migrate_table_2402/migrate_local_table_to_2402.sh index 5a9679b..6c75784 100644 --- a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/migrate_table_2402/migrate_local_table_to_2402.sh +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/migrate_table_2402/migrate_local_table_to_2402.sh @@ -15,26 +15,26 @@ timestamp_start=`date --utc --date="$data_start_time" +%s` timestamp_end=`date --utc --date="$data_end_time" +%s` # 需要迁移的表名称 -session_record_table_src="tsg_galaxy_v3.session_record_local_old" -session_record_table_dest="tsg_galaxy_v3.session_record_local" +session_record_table_src="tsg_galaxy_tmp.session_record_local_old" +session_record_table_dest="tsg_galaxy_tmp.session_record_local" -security_event_table_src="tsg_galaxy_v3.security_event_local_old" -security_event_table_dest="tsg_galaxy_v3.security_event_local" +security_event_table_src="tsg_galaxy_tmp.security_event_local_old" +security_event_table_dest="tsg_galaxy_tmp.security_event_local" -monitor_event_table_src="tsg_galaxy_v3.security_event_local_old" -monitor_event_table_dest="tsg_galaxy_v3.monitor_event_local" +monitor_event_table_src="tsg_galaxy_tmp.security_event_local_old" +monitor_event_table_dest="tsg_galaxy_tmp.monitor_event_local" -transaction_record_table_src="tsg_galaxy_v3.transaction_record_local_old" -transaction_record_table_dest="tsg_galaxy_v3.transaction_record_local" +transaction_record_table_src="tsg_galaxy_tmp.transaction_record_local_old" +transaction_record_table_dest="tsg_galaxy_tmp.transaction_record_local" -voip_record_table_src="tsg_galaxy_v3.voip_record_local_old" -voip_record_table_dest="tsg_galaxy_v3.voip_record_local" +voip_record_table_src="tsg_galaxy_tmp.voip_record_local_old" +voip_record_table_dest="tsg_galaxy_tmp.voip_record_local" -proxy_event_table_src="tsg_galaxy_v3.proxy_event_local_old" -proxy_event_table_dest="tsg_galaxy_v3.proxy_event_local" +proxy_event_table_src="tsg_galaxy_tmp.proxy_event_local_old" +proxy_event_table_dest="tsg_galaxy_tmp.proxy_event_local" -dos_event_table_src="tsg_galaxy_v3.dos_event_local_old" -dos_event_table_dest="tsg_galaxy_v3.dos_event_local" +dos_event_table_src="tsg_galaxy_tmp.dos_event_local_old" +dos_event_table_dest="tsg_galaxy_tmp.dos_event_local" # ck客户端参数 ip="127.0.0.1" diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_tmp_2307_to_tmp_2307_old.sql b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_tmp_2307_to_tmp_2307_old.sql new file mode 100644 index 0000000..72ae3ef --- /dev/null +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_tmp_2307_to_tmp_2307_old.sql @@ -0,0 +1,53 @@ +set distributed_ddl_task_timeout = 180; + +-- 删除源表同步子表物化视图 + + +-- 源表rename到历史表 +RENAME TABLE tsg_galaxy_tmp.session_record_local to tsg_galaxy_tmp.session_record_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_tmp.security_event_local to tsg_galaxy_tmp.security_event_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_tmp.transaction_record_local to tsg_galaxy_tmp.transaction_record_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_tmp.voip_record_local to tsg_galaxy_tmp.voip_record_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_tmp.proxy_event_local to tsg_galaxy_tmp.proxy_event_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_tmp.dos_event_local to tsg_galaxy_tmp.dos_event_local_old on cluster ck_cluster; + +drop table if exists tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query; +drop table if exists tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query; +drop table if exists tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query; +drop table if exists tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query; +drop table if exists tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query; +drop table if exists tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query; + + +-- 创建源分布式表old +create table IF NOT EXISTS tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local_old,rand()); + +create table IF NOT EXISTS tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local_old,rand()); + +create table IF NOT EXISTS tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query( + log_id UInt64, + profile_id UInt64, + start_time Int64 +) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local_old,rand()); + diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_2307_to_tmp_2307.sql b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_to_tmp.sql similarity index 78% rename from Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_2307_to_tmp_2307.sql rename to Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_to_tmp.sql index 62fdbde..0cfd656 100644 --- a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_2307_to_tmp_2307.sql +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/move_v3_to_tmp.sql @@ -1,46 +1,47 @@ set distributed_ddl_task_timeout = 180; -- 删除源表同步子表物化视图 - +create database if not exists tsg_galaxy_tmp on cluster ck_cluster; +create database if not exists tsg_galaxy_tmp on cluster ck_query; -- 源表rename到历史表 -RENAME TABLE tsg_galaxy_v3.session_record_local to tsg_galaxy_tmp.session_record_local_old on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.security_event_local to tsg_galaxy_tmp.security_event_local_old on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.transaction_record_local to tsg_galaxy_tmp.transaction_record_local_old on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.voip_record_local to tsg_galaxy_tmp.voip_record_local_old on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.proxy_event_local to tsg_galaxy_tmp.proxy_event_local_old on cluster ck_cluster; -RENAME TABLE tsg_galaxy_v3.dos_event_local to tsg_galaxy_tmp.dos_event_local_old on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.session_record_local to tsg_galaxy_tmp.session_record_local on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.security_event_local to tsg_galaxy_tmp.security_event_local on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.transaction_record_local to tsg_galaxy_tmp.transaction_record_local on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.voip_record_local to tsg_galaxy_tmp.voip_record_local on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.proxy_event_local to tsg_galaxy_tmp.proxy_event_local on cluster ck_cluster; +RENAME TABLE tsg_galaxy_v3.dos_event_local to tsg_galaxy_tmp.dos_event_local on cluster ck_cluster; -- 创建源分布式表old create table IF NOT EXISTS tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query ( common_recv_time Int64, common_log_id UInt64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local_old,rand()); +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local,rand()); create table IF NOT EXISTS tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query ( common_recv_time Int64, common_log_id UInt64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local_old,rand()); +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local,rand()); CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query( common_recv_time Int64, common_log_id UInt64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local_old,rand()); +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local,rand()); CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query( common_recv_time Int64, common_log_id UInt64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local_old,rand()); +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local,rand()); create table IF NOT EXISTS tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query ( common_recv_time Int64, common_log_id UInt64 -) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local_old,rand()); +) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local,rand()); CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query( log_id UInt64, profile_id UInt64, start_time Int64 -) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local_old,rand()); +) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local,rand()); diff --git a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md index afd84c3..02271f0 100644 --- a/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md +++ b/Clickhouse最新全量建表语句/通用跨版本迁移数据步骤/通用跨版本迁移数据步骤.md @@ -1,70 +1,84 @@ -由于各环境当前使用tsg版本与升级的版本均不同，故在此提供通用步骤，因主键与字段不同，旧版本统一升级至23.07版本进行处理 +由于各环境当前使用tsg版本与升级的版本均不同,故在此提供通用步骤,因主键与字段不同,旧版本统一升级至23.07版本进行处理,为及时接入数据,离线迁移操作在tmp数据库进行。 具体步骤： Step1 ：停止入库任务。 -Step2 ：旧版本clickhouse库表升级到23.07版本 -Step3 ：tsg_galaxy_tmp数据库新建24.02版本库表,修改建表语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp -Step4 ：23.07版本库表迁移至tsg_galaxy_tmp数据库 -Step5 : tsg_galaxy_v3数据库新建目标版本库表（如24.04） -Step6 : 启动入库任务->tsg_galaxy_v3 +Step2 ：旧版本clickhouse库表迁移至tsg_galaxy_tmp,验证成功后删除tsg_galaxy_v3数据库 +Step3 : tsg_galaxy_v3数据库新建目标版本库表（如24.04） +Step4 : 启动入库任务->tsg_galaxy_v3 +Step5 : tsg_galaxy_tmp数据库表升级到23.07修改升级语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp,升级验证成功后重命名为old表。 +Step6 ：tsg_galaxy_tmp数据库新建24.02版本库表,修改建表语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp Step7 : 迁移脚本迁移tsg_galaxy_tmp(23.07)->tsg_galaxy_tmp(24.02) Step8 : 升级tsg_galaxy_tmp(24.02)->升级tsg_galaxy_tmp目标版本如（24.04） Step9 : 命令迁移tsg_galaxy_tmp目标版本如（24.04）->tsg_galaxy_v3目标版本如（24.04）,按照（天）partition手动迁移 说明 -请按步骤依次执行，执行脚本报错时联系研发处理后再执行之后的步骤。 +请按步骤依次执行,执行脚本报错时联系研发处理后再执行之后的步骤。 所有ck步骤都需要在query节点执行 -执行所有sql语句之前需要停止日志留存调度任务，确保ck中无分布式ddl语句H执行，否则执行的sql会阻塞住，影响后续步骤执行 -验证sql需要在query节点执行 +执行所有sql语句之前需要停止日志留存调度任务,确保ck中无分布式ddl语句执行,否则执行的sql会阻塞住,影响后续步骤执行 +验证sql需要在所有query节点执行 clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1" -若返回结果为空则可执行升级步骤，否则需要等待。 +若返回结果为空则可执行升级步骤,否则需要等待。 一、停止旧表ck入库任务 停止旧表ck入库任务 -二、旧版本clickhouse库表升级到23.07版本,依次执行版本升级语句 +二、旧版本clickhouse库表迁移至tsg_galaxy_tmp + +1.查看迁移前v3库表数据量 + +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_v3_old_table_row_count.sql -三、临时库初始化24.02版本库表 - -1.执行2402版本初始化建表语句 +2.执行迁移sql +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_v3_to_tmp.sql -clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < init_tsg_galaxy_tmp_24_02_table.sql - - - -2.校验表结构 - - -clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < check_tsg_galaxy_tmp_24_02_table.sql - - -无报错信息说明校验通过 - -四、23.07版本库表迁移至tsg_galaxy_tmp数据库 - -1.迁移sql -clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_v3_2307_to_tmp_2307.sql - - -2.查看tmp库old表数据量 +3.查看tmp库old表数据量 clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_tmp_old_table_row_count.sql -五、tsg_galaxy_v3数据库新建目标版本库表 +4.数据量与原数据量一致,可删除数据库tsg_galaxy_v3。 + +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 --query "drop database if exists tsg_galaxy_v3 on cluster ck_cluster " +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 --query "drop database if exists tsg_galaxy_v3 on cluster ck_query " + +三、tsg_galaxy_v3数据库新建目标版本库表并进行校验 clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 对应版本初始化sql -六、启动ck入库任务 +四、启动ck入库任务 1.启动目标版本ck入库任务 +五、tsg_galaxy_tmp数据库表升级到23.07,升级验证成功后重命名为old表 + +1.tsg_galaxy_tmp旧版本clickhouse库表升级到23.07版本（注意所有版本升级sql需要将tsg_galaxy_v3替换tsg_galaxy_tmp,包括校验sql）,依次执行版本升级语句,并进行校验 +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 各版本升级sql + +2.验证成功后重命名为old表 +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_tmp_2307_to_tmp_2307_old.sql + + +3.查看tmp库old表数据量 + +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_tmp_old_table_row_count.sql + + +六、临时库初始化24.02版本库表 + +1.执行2402版本初始化建表语句 +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < init_tsg_galaxy_tmp_24_02_table.sql + +2.校验表结构 +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < check_tsg_galaxy_tmp_24_02_table.sql +无报错信息说明校验通过 + + 七、离线脚本同步历史数据至临时数据库2402版本库表 -在query节点执行以下步骤，iplist.txt中为ck所有data节点ip地址。 +在query节点执行以下步骤,iplist.txt中为ck所有data节点ip地址。 步骤描述： 1.进入migrate_table_2402文件夹,使脚本可执行 @@ -81,7 +95,7 @@ chmod +x ./*.sh -2.选择迁移某个表，同步需要时间区间的数据，时间区间:实时同步任务开始时间向前推n天, 实时同步任务开始时间)，时间区间为左闭右开，不包含结束时间点。 +2.选择迁移某个表,同步需要时间区间的数据,时间区间:实时同步任务开始时间向前推n天, 实时同步任务开始时间),时间区间为左闭右开,不包含结束时间点。 # 迁移security_event表 @@ -89,7 +103,7 @@ chmod +x ./*.sh -3.监控data节点迁移情况，所有表迁移完成后，确认每个节点同步数据成功/失败批次数，如有失败批次确认是否需要处理 +3.监控data节点迁移情况,所有表迁移完成后,确认每个节点同步数据成功/失败批次数,如有失败批次确认是否需要处理 # 监控security_event表迁移 @@ -97,7 +111,7 @@ chmod +x ./*.sh -4.选择下个张需要迁移的表，重复2-4步骤。支持选择迁移的表有: security_event, monitor_event, session_record, transaction_record, voip_record, proxy_event, dos_event。 +4.选择下个张需要迁移的表,重复2-4步骤。支持选择迁移的表有: security_event, monitor_event, session_record, transaction_record, voip_record, proxy_event, dos_event。 迁移和监控各个表执行命令示例： @@ -143,8 +157,8 @@ chmod +x ./*.sh ./03_monitor_migrate_table.sh dos_event -迁移日志无报错，数据迁移完成。 -如果有数据迁移失败批次，查看新老表迁移数据量对应情况(ck每台data节点)： +迁移日志无报错,数据迁移完成。 +如果有数据迁移失败批次,查看新老表迁移数据量对应情况(ck每台data节点)： -- security_event