From d77b3f1e97c7370c51113a1e51216985efd52e76 Mon Sep 17 00:00:00 2001 From: lifengchao Date: Sun, 7 Apr 2024 19:26:56 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=9F=A5=E8=AF=A2old?= =?UTF-8?q?=E8=A1=A8=E6=95=B0=E6=8D=AE=E8=A1=8C=E6=95=B0SQL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../01_cat_old_table_row_count.sql | 11 +++++++ .../01_rename_old_table.sql | 33 +++++++++++++++++++ .../06_drop_old_table.sql | 17 ++++++++++ .../TSG 2307版本到2402版本升级操作/ck表升级步骤.md | 19 ++++++++++- 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_cat_old_table_row_count.sql create mode 100644 Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/06_drop_old_table.sql diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_cat_old_table_row_count.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_cat_old_table_row_count.sql new file mode 100644 index 0000000..e506a1c --- /dev/null +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_cat_old_table_row_count.sql @@ -0,0 +1,11 @@ +select 'session_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.session_record_old; + +select 'security_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.security_event_old; + +select 'transaction_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.transaction_record_old; + +select 'voip_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.voip_record_old; + +select 'proxy_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.proxy_event_old; + +select 'dos_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.dos_event_old; \ No newline at end of file diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql index 99e55ba..360a46a 100644 --- a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql @@ -72,3 +72,36 @@ drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query; drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster; drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query; drop table IF EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster; + +-- 创建源码分布式表old +create table IF NOT EXISTS tsg_galaxy_v3.session_record_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local_old,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.security_event_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_old ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_old ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local_old,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event_old ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local_old,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_old ON CLUSTER ck_query( + log_id UInt64, + profile_id UInt64, + start_time Int64 +) ENGINE = Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local_old,rand()); + diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/06_drop_old_table.sql b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/06_drop_old_table.sql new file mode 100644 index 0000000..4a22825 --- /dev/null +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/06_drop_old_table.sql @@ -0,0 +1,17 @@ +set distributed_ddl_task_timeout = 180; + +-- 删除旧表 +drop table IF EXISTS tsg_galaxy_v3.session_record_local_old on cluster ck_cluster; +drop table IF EXISTS tsg_galaxy_v3.security_event_local_old on cluster ck_cluster; +drop table IF EXISTS tsg_galaxy_v3.transaction_record_local_old on cluster ck_cluster; +drop table IF EXISTS tsg_galaxy_v3.voip_record_local_old on cluster ck_cluster; +drop table IF EXISTS tsg_galaxy_v3.proxy_event_local_old on cluster ck_cluster; +drop table IF EXISTS tsg_galaxy_v3.dos_event_local_old on cluster ck_cluster; + +-- 删除旧表分布式表 +DROP TABLE IF EXISTS tsg_galaxy_v3.session_record_old ON CLUSTER ck_query; +DROP TABLE IF EXISTS tsg_galaxy_v3.security_event_old ON CLUSTER ck_query; +DROP TABLE IF EXISTS tsg_galaxy_v3.transaction_record_old ON CLUSTER ck_query; +DROP TABLE IF EXISTS tsg_galaxy_v3.voip_record_old ON CLUSTER ck_query; +DROP TABLE IF EXISTS tsg_galaxy_v3.proxy_event_old ON CLUSTER ck_query; +DROP TABLE IF EXISTS tsg_galaxy_v3.dos_event_old ON CLUSTER ck_query; diff --git a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md index 1a86a97..5614172 100644 --- a/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md +++ b/Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md @@ -24,11 +24,16 @@ clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --que # 二、旧表重命名为历史表 -重命名旧表, 删除废弃表 +* 1.重命名旧表, 删除废弃表 ```sql clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_rename_old_table.sql ``` +* 2.查看旧表数据量 +``` +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_cat_old_table_row_count.sql +``` + # 三、初始化新表 * 1.执行2402版本初始化建表语句 @@ -65,6 +70,11 @@ tsg_galaxy_v3.proxy_event_local_old tsg_galaxy_v3.dos_event_local_old ``` +* 3.查看旧表是否有数据入库(间隔一段时间查询,后面查询比前面查询行数多说明有数据写入) +``` +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_cat_old_table_row_count.sql +``` + # 六、各个数据中心全部升级完成后停止旧表ck入库任务 * 1.升级各个数据中心，各个数据中心全部升级完成后，停止旧表ck入库任务(如果启动的话) @@ -295,3 +305,10 @@ group by date_trunc('day', toDateTime(start_time)) order by d ; ``` + +# 八、删除旧表 +数据迁移完成后，不需要查看旧表时，删除旧表。 + +```sh +clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 06_drop_old_table.sql +``` \ No newline at end of file