修改迁移步骤，将耗时的升级操作改为tmp数据库进行
This commit is contained in:
wangkuan
@@ -0,0 +1,11 @@
|
||||
select 'session_record' as table_name, count(*) as cnt from tsg_galaxy_v3.session_record;
|
||||
|
||||
select 'security_event' as table_name, count(*) as cnt from tsg_galaxy_v3.security_event;
|
||||
|
||||
select 'transaction_record' as table_name, count(*) as cnt from tsg_galaxy_v3.transaction_record;
|
||||
|
||||
select 'voip_record' as table_name, count(*) as cnt from tsg_galaxy_v3.voip_record;
|
||||
|
||||
select 'proxy_event' as table_name, count(*) as cnt from tsg_galaxy_v3.proxy_event;
|
||||
|
||||
select 'dos_event' as table_name, count(*) as cnt from tsg_galaxy_v3.dos_event;
|
||||
@@ -15,26 +15,26 @@ timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 需要迁移的表名称
|
||||
session_record_table_src="tsg_galaxy_v3.session_record_local_old"
|
||||
session_record_table_dest="tsg_galaxy_v3.session_record_local"
|
||||
session_record_table_src="tsg_galaxy_tmp.session_record_local_old"
|
||||
session_record_table_dest="tsg_galaxy_tmp.session_record_local"
|
||||
|
||||
security_event_table_src="tsg_galaxy_v3.security_event_local_old"
|
||||
security_event_table_dest="tsg_galaxy_v3.security_event_local"
|
||||
security_event_table_src="tsg_galaxy_tmp.security_event_local_old"
|
||||
security_event_table_dest="tsg_galaxy_tmp.security_event_local"
|
||||
|
||||
monitor_event_table_src="tsg_galaxy_v3.security_event_local_old"
|
||||
monitor_event_table_dest="tsg_galaxy_v3.monitor_event_local"
|
||||
monitor_event_table_src="tsg_galaxy_tmp.security_event_local_old"
|
||||
monitor_event_table_dest="tsg_galaxy_tmp.monitor_event_local"
|
||||
|
||||
transaction_record_table_src="tsg_galaxy_v3.transaction_record_local_old"
|
||||
transaction_record_table_dest="tsg_galaxy_v3.transaction_record_local"
|
||||
transaction_record_table_src="tsg_galaxy_tmp.transaction_record_local_old"
|
||||
transaction_record_table_dest="tsg_galaxy_tmp.transaction_record_local"
|
||||
|
||||
voip_record_table_src="tsg_galaxy_v3.voip_record_local_old"
|
||||
voip_record_table_dest="tsg_galaxy_v3.voip_record_local"
|
||||
voip_record_table_src="tsg_galaxy_tmp.voip_record_local_old"
|
||||
voip_record_table_dest="tsg_galaxy_tmp.voip_record_local"
|
||||
|
||||
proxy_event_table_src="tsg_galaxy_v3.proxy_event_local_old"
|
||||
proxy_event_table_dest="tsg_galaxy_v3.proxy_event_local"
|
||||
proxy_event_table_src="tsg_galaxy_tmp.proxy_event_local_old"
|
||||
proxy_event_table_dest="tsg_galaxy_tmp.proxy_event_local"
|
||||
|
||||
dos_event_table_src="tsg_galaxy_v3.dos_event_local_old"
|
||||
dos_event_table_dest="tsg_galaxy_v3.dos_event_local"
|
||||
dos_event_table_src="tsg_galaxy_tmp.dos_event_local_old"
|
||||
dos_event_table_dest="tsg_galaxy_tmp.dos_event_local"
|
||||
|
||||
# ck客户端参数
|
||||
ip="127.0.0.1"
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- 删除源表同步子表物化视图
|
||||
|
||||
|
||||
-- 源表rename到历史表
|
||||
RENAME TABLE tsg_galaxy_tmp.session_record_local to tsg_galaxy_tmp.session_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.security_event_local to tsg_galaxy_tmp.security_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.transaction_record_local to tsg_galaxy_tmp.transaction_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.voip_record_local to tsg_galaxy_tmp.voip_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.proxy_event_local to tsg_galaxy_tmp.proxy_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_tmp.dos_event_local to tsg_galaxy_tmp.dos_event_local_old on cluster ck_cluster;
|
||||
|
||||
drop table if exists tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query;
|
||||
drop table if exists tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query;
|
||||
|
||||
|
||||
-- 创建源分布式表old
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local_old,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local_old,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query(
|
||||
log_id UInt64,
|
||||
profile_id UInt64,
|
||||
start_time Int64
|
||||
) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local_old,rand());
|
||||
|
||||
@@ -1,46 +1,47 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- 删除源表同步子表物化视图
|
||||
|
||||
create database if not exists tsg_galaxy_tmp on cluster ck_cluster;
|
||||
create database if not exists tsg_galaxy_tmp on cluster ck_query;
|
||||
|
||||
-- 源表rename到历史表
|
||||
RENAME TABLE tsg_galaxy_v3.session_record_local to tsg_galaxy_tmp.session_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.security_event_local to tsg_galaxy_tmp.security_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.transaction_record_local to tsg_galaxy_tmp.transaction_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.voip_record_local to tsg_galaxy_tmp.voip_record_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.proxy_event_local to tsg_galaxy_tmp.proxy_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.dos_event_local to tsg_galaxy_tmp.dos_event_local_old on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.session_record_local to tsg_galaxy_tmp.session_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.security_event_local to tsg_galaxy_tmp.security_event_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.transaction_record_local to tsg_galaxy_tmp.transaction_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.voip_record_local to tsg_galaxy_tmp.voip_record_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.proxy_event_local to tsg_galaxy_tmp.proxy_event_local on cluster ck_cluster;
|
||||
RENAME TABLE tsg_galaxy_v3.dos_event_local to tsg_galaxy_tmp.dos_event_local on cluster ck_cluster;
|
||||
|
||||
|
||||
-- 创建源分布式表old
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.session_record_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local_old,rand());
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,session_record_local,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.security_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local_old,rand());
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,security_event_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local_old,rand());
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,transaction_record_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.voip_record_old ON CLUSTER ck_query(
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local_old,rand());
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,voip_record_local,rand());
|
||||
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.proxy_event_old ON CLUSTER ck_query (
|
||||
common_recv_time Int64,
|
||||
common_log_id UInt64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local_old,rand());
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,proxy_event_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_query(
|
||||
log_id UInt64,
|
||||
profile_id UInt64,
|
||||
start_time Int64
|
||||
) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local_old,rand());
|
||||
) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local,rand());
|
||||
|
||||
@@ -1,70 +1,84 @@
|
||||
由于各环境当前使用tsg版本与升级的版本均不同，故在此提供通用步骤，因主键与字段不同，旧版本统一升级至23.07版本进行处理
|
||||
由于各环境当前使用tsg版本与升级的版本均不同,故在此提供通用步骤,因主键与字段不同,旧版本统一升级至23.07版本进行处理,为及时接入数据,离线迁移操作在tmp数据库进行。
|
||||
具体步骤：
|
||||
|
||||
Step1 ：停止入库任务。
|
||||
Step2 ：旧版本clickhouse库表升级到23.07版本
|
||||
Step3 ：tsg_galaxy_tmp数据库新建24.02版本库表,修改建表语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp
|
||||
Step4 ：23.07版本库表迁移至tsg_galaxy_tmp数据库
|
||||
Step5 : tsg_galaxy_v3数据库新建目标版本库表（如24.04）
|
||||
Step6 : 启动入库任务->tsg_galaxy_v3
|
||||
Step2 ：旧版本clickhouse库表迁移至tsg_galaxy_tmp,验证成功后删除tsg_galaxy_v3数据库
|
||||
Step3 : tsg_galaxy_v3数据库新建目标版本库表（如24.04）
|
||||
Step4 : 启动入库任务->tsg_galaxy_v3
|
||||
Step5 : tsg_galaxy_tmp数据库表升级到23.07修改升级语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp,升级验证成功后重命名为old表。
|
||||
Step6 ：tsg_galaxy_tmp数据库新建24.02版本库表,修改建表语句中数据库名tsg_galaxy_v3->tsg_galaxy_tmp
|
||||
Step7 : 迁移脚本迁移tsg_galaxy_tmp(23.07)->tsg_galaxy_tmp(24.02)
|
||||
Step8 : 升级tsg_galaxy_tmp(24.02)->升级tsg_galaxy_tmp目标版本如（24.04）
|
||||
Step9 : 命令迁移tsg_galaxy_tmp目标版本如（24.04）->tsg_galaxy_v3目标版本如（24.04）,按照（天）partition手动迁移
|
||||
|
||||
说明
|
||||
|
||||
请按步骤依次执行，执行脚本报错时联系研发处理后再执行之后的步骤。
|
||||
请按步骤依次执行,执行脚本报错时联系研发处理后再执行之后的步骤。
|
||||
所有ck步骤都需要在query节点执行
|
||||
执行所有sql语句之前需要停止日志留存调度任务，确保ck中无分布式ddl语句H执行，否则执行的sql会阻塞住，影响后续步骤执行
|
||||
验证sql需要在query节点执行
|
||||
执行所有sql语句之前需要停止日志留存调度任务,确保ck中无分布式ddl语句执行,否则执行的sql会阻塞住,影响后续步骤执行
|
||||
验证sql需要在所有query节点执行
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1"
|
||||
若返回结果为空则可执行升级步骤，否则需要等待。
|
||||
若返回结果为空则可执行升级步骤,否则需要等待。
|
||||
|
||||
|
||||
一、停止旧表ck入库任务
|
||||
停止旧表ck入库任务
|
||||
|
||||
二、旧版本clickhouse库表升级到23.07版本,依次执行版本升级语句
|
||||
二、旧版本clickhouse库表迁移至tsg_galaxy_tmp
|
||||
|
||||
1.查看迁移前v3库表数据量
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_v3_old_table_row_count.sql
|
||||
|
||||
|
||||
三、临时库初始化24.02版本库表
|
||||
|
||||
1.执行2402版本初始化建表语句
|
||||
2.执行迁移sql
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_v3_to_tmp.sql
|
||||
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < init_tsg_galaxy_tmp_24_02_table.sql
|
||||
|
||||
|
||||
|
||||
2.校验表结构
|
||||
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < check_tsg_galaxy_tmp_24_02_table.sql
|
||||
|
||||
|
||||
无报错信息说明校验通过
|
||||
|
||||
四、23.07版本库表迁移至tsg_galaxy_tmp数据库
|
||||
|
||||
1.迁移sql
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_v3_2307_to_tmp_2307.sql
|
||||
|
||||
|
||||
2.查看tmp库old表数据量
|
||||
3.查看tmp库old表数据量
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_tmp_old_table_row_count.sql
|
||||
|
||||
五、tsg_galaxy_v3数据库新建目标版本库表
|
||||
4.数据量与原数据量一致,可删除数据库tsg_galaxy_v3。
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 --query "drop database if exists tsg_galaxy_v3 on cluster ck_cluster "
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 --query "drop database if exists tsg_galaxy_v3 on cluster ck_query "
|
||||
|
||||
三、tsg_galaxy_v3数据库新建目标版本库表并进行校验
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 对应版本初始化sql
|
||||
|
||||
六、启动ck入库任务
|
||||
四、启动ck入库任务
|
||||
|
||||
1.启动目标版本ck入库任务
|
||||
|
||||
|
||||
五、tsg_galaxy_tmp数据库表升级到23.07,升级验证成功后重命名为old表
|
||||
|
||||
1.tsg_galaxy_tmp旧版本clickhouse库表升级到23.07版本（注意所有版本升级sql需要将tsg_galaxy_v3替换tsg_galaxy_tmp,包括校验sql）,依次执行版本升级语句,并进行校验
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 各版本升级sql
|
||||
|
||||
2.验证成功后重命名为old表
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_tmp_2307_to_tmp_2307_old.sql
|
||||
|
||||
|
||||
3.查看tmp库old表数据量
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_tmp_old_table_row_count.sql
|
||||
|
||||
|
||||
六、临时库初始化24.02版本库表
|
||||
|
||||
1.执行2402版本初始化建表语句
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < init_tsg_galaxy_tmp_24_02_table.sql
|
||||
|
||||
2.校验表结构
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < check_tsg_galaxy_tmp_24_02_table.sql
|
||||
无报错信息说明校验通过
|
||||
|
||||
|
||||
七、离线脚本同步历史数据至临时数据库2402版本库表
|
||||
在query节点执行以下步骤，iplist.txt中为ck所有data节点ip地址。
|
||||
在query节点执行以下步骤,iplist.txt中为ck所有data节点ip地址。
|
||||
步骤描述：
|
||||
|
||||
1.进入migrate_table_2402文件夹,使脚本可执行
|
||||
@@ -81,7 +95,7 @@ chmod +x ./*.sh
|
||||
|
||||
|
||||
|
||||
2.选择迁移某个表，同步需要时间区间的数据，时间区间:实时同步任务开始时间向前推n天, 实时同步任务开始时间)，时间区间为左闭右开，不包含结束时间点。
|
||||
2.选择迁移某个表,同步需要时间区间的数据,时间区间:实时同步任务开始时间向前推n天, 实时同步任务开始时间),时间区间为左闭右开,不包含结束时间点。
|
||||
|
||||
|
||||
# 迁移security_event表
|
||||
@@ -89,7 +103,7 @@ chmod +x ./*.sh
|
||||
|
||||
|
||||
|
||||
3.监控data节点迁移情况，所有表迁移完成后，确认每个节点同步数据成功/失败批次数，如有失败批次确认是否需要处理
|
||||
3.监控data节点迁移情况,所有表迁移完成后,确认每个节点同步数据成功/失败批次数,如有失败批次确认是否需要处理
|
||||
|
||||
|
||||
# 监控security_event表迁移
|
||||
@@ -97,7 +111,7 @@ chmod +x ./*.sh
|
||||
|
||||
|
||||
|
||||
4.选择下个张需要迁移的表，重复2-4步骤。支持选择迁移的表有: security_event, monitor_event, session_record, transaction_record, voip_record, proxy_event, dos_event。
|
||||
4.选择下个张需要迁移的表,重复2-4步骤。支持选择迁移的表有: security_event, monitor_event, session_record, transaction_record, voip_record, proxy_event, dos_event。
|
||||
|
||||
迁移和监控各个表执行命令示例：
|
||||
|
||||
@@ -143,8 +157,8 @@ chmod +x ./*.sh
|
||||
./03_monitor_migrate_table.sh dos_event
|
||||
|
||||
|
||||
迁移日志无报错，数据迁移完成。
|
||||
如果有数据迁移失败批次，查看新老表迁移数据量对应情况(ck每台data节点)：
|
||||
迁移日志无报错,数据迁移完成。
|
||||
如果有数据迁移失败批次,查看新老表迁移数据量对应情况(ck每台data节点)：
|
||||
|
||||
-- security_event
|
||||
|
||||
|
||||
Reference in New Issue
Block a user