gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加查询old表数据行数SQL

Browse Source
This commit is contained in:
lifengchao
2024-04-07 19:26:56 +08:00
parent 8c3d754207
commit d77b3f1e97
4 changed files with 79 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_cat_old_table_row_count.sql Normal file
View File

@@ -0,0 +1,11 @@
select 'session_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.session_record_old;
select 'security_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.security_event_old;
select 'transaction_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.transaction_record_old;
select 'voip_record_old' as table_name, count(*) as cnt from tsg_galaxy_v3.voip_record_old;
select 'proxy_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.proxy_event_old;
select 'dos_event_old' as table_name, count(*) as cnt from tsg_galaxy_v3.dos_event_old;

33
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql
View File

@@ -72,3 +72,36 @@ drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query;
drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster; drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster;
drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query; drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query;
drop table IF EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster; drop table IF EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster;
-- 创建源码分布式表old
create table IF NOT EXISTS tsg_galaxy_v3.session_record_old ON CLUSTER ck_query (
common_recv_time Int64,
common_log_id UInt64
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local_old,rand());
create table IF NOT EXISTS tsg_galaxy_v3.security_event_old ON CLUSTER ck_query (
common_recv_time Int64,
common_log_id UInt64
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local_old,rand());
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_old ON CLUSTER ck_query(
common_recv_time Int64,
common_log_id UInt64
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local_old,rand());
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_old ON CLUSTER ck_query(
common_recv_time Int64,
common_log_id UInt64
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local_old,rand());
create table IF NOT EXISTS tsg_galaxy_v3.proxy_event_old ON CLUSTER ck_query (
common_recv_time Int64,
common_log_id UInt64
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local_old,rand());
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_old ON CLUSTER ck_query(
log_id UInt64,
profile_id UInt64,
start_time Int64
) ENGINE = Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local_old,rand());

17
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/06_drop_old_table.sql Normal file
View File

@@ -0,0 +1,17 @@
set distributed_ddl_task_timeout = 180;
-- 删除旧表
drop table IF EXISTS tsg_galaxy_v3.session_record_local_old on cluster ck_cluster;
drop table IF EXISTS tsg_galaxy_v3.security_event_local_old on cluster ck_cluster;
drop table IF EXISTS tsg_galaxy_v3.transaction_record_local_old on cluster ck_cluster;
drop table IF EXISTS tsg_galaxy_v3.voip_record_local_old on cluster ck_cluster;
drop table IF EXISTS tsg_galaxy_v3.proxy_event_local_old on cluster ck_cluster;
drop table IF EXISTS tsg_galaxy_v3.dos_event_local_old on cluster ck_cluster;
-- 删除旧表分布式表
DROP TABLE IF EXISTS tsg_galaxy_v3.session_record_old ON CLUSTER ck_query;
DROP TABLE IF EXISTS tsg_galaxy_v3.security_event_old ON CLUSTER ck_query;
DROP TABLE IF EXISTS tsg_galaxy_v3.transaction_record_old ON CLUSTER ck_query;
DROP TABLE IF EXISTS tsg_galaxy_v3.voip_record_old ON CLUSTER ck_query;
DROP TABLE IF EXISTS tsg_galaxy_v3.proxy_event_old ON CLUSTER ck_query;
DROP TABLE IF EXISTS tsg_galaxy_v3.dos_event_old ON CLUSTER ck_query;

19
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md
View File

@@ -24,11 +24,16 @@ clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --que
# 二、旧表重命名为历史表 # 二、旧表重命名为历史表
重命名旧表, 删除废弃表 * 1.重命名旧表, 删除废弃表
```sql ```sql
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_rename_old_table.sql clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_rename_old_table.sql
``` ```
* 2.查看旧表数据量
```
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_cat_old_table_row_count.sql
```
# 三、初始化新表 # 三、初始化新表
* 1.执行2402版本初始化建表语句 * 1.执行2402版本初始化建表语句
@@ -65,6 +70,11 @@ tsg_galaxy_v3.proxy_event_local_old
tsg_galaxy_v3.dos_event_local_old tsg_galaxy_v3.dos_event_local_old
``` ```
* 3.查看旧表是否有数据入库(间隔一段时间查询,后面查询比前面查询行数多说明有数据写入)
```
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_cat_old_table_row_count.sql
```
# 六、各个数据中心全部升级完成后停止旧表ck入库任务 # 六、各个数据中心全部升级完成后停止旧表ck入库任务
* 1.升级各个数据中心，各个数据中心全部升级完成后，停止旧表ck入库任务(如果启动的话) * 1.升级各个数据中心，各个数据中心全部升级完成后，停止旧表ck入库任务(如果启动的话)
@@ -295,3 +305,10 @@ group by date_trunc('day', toDateTime(start_time))
order by d order by d
; ;
``` ```
# 八、删除旧表
数据迁移完成后，不需要查看旧表时，删除旧表。
```sh
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 06_drop_old_table.sql
```