gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG 2307版本到2402版本升级步骤变更：先重命名旧表和初始化新表数据库

Browse Source
This commit is contained in:
lifengchao
2024-03-22 11:33:24 +08:00
parent 951ce0b189
commit 42af770e19
7 changed files with 75 additions and 2068 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1995
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_create_table_2402.sql
View File

File diff suppressed because it is too large Load Diff

22
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_rename_table.sql → Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/01_rename_old_table.sql
View File

@@ -1,14 +1,5 @@
set distributed_ddl_task_timeout = 180; set distributed_ddl_task_timeout = 180;
-- 删除源表同步到临时表物化视图, 七个表
drop view if exists tsg_galaxy_v3.session_record_local_2307_to_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.security_event_local_2307_to_security_event_local_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.security_event_local_2307_to_monitor_event_local_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.transaction_record_local_2307_to_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.voip_record_local_2307_to_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.proxy_event_local_2307_to_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.dos_event_local_2307_to_2402_view on cluster ck_cluster;
-- 删除源表同步子表物化视图 -- 删除源表同步子表物化视图
drop VIEW IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster; drop VIEW IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster;
drop VIEW IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster; drop VIEW IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster;
@@ -81,16 +72,3 @@ drop table IF EXISTS tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query;
drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster; drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_cluster;
drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query; drop table IF EXISTS tsg_galaxy_v3.active_defence_event ON CLUSTER ck_query;
drop table IF EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster; drop table IF EXISTS tsg_galaxy_v3.active_defence_event_local ON CLUSTER ck_cluster;
-- 删除临时表之间物化视图
drop VIEW IF EXISTS tsg_galaxy_v3.security_event_materialized_view_2402 ON CLUSTER ck_cluster;
drop VIEW IF EXISTS tsg_galaxy_v3.monitor_event_materialized_view_2402 ON CLUSTER ck_cluster;
-- 临时表rename到目标表
RENAME TABLE tsg_galaxy_v3.session_record_local_2402 to tsg_galaxy_v3.session_record_local on cluster ck_cluster;
RENAME TABLE tsg_galaxy_v3.security_event_local_2402 to tsg_galaxy_v3.security_event_local on cluster ck_cluster;
RENAME TABLE tsg_galaxy_v3.monitor_event_local_2402 to tsg_galaxy_v3.monitor_event_local on cluster ck_cluster;
RENAME TABLE tsg_galaxy_v3.transaction_record_local_2402 to tsg_galaxy_v3.transaction_record_local on cluster ck_cluster;
RENAME TABLE tsg_galaxy_v3.voip_record_local_2402 to tsg_galaxy_v3.voip_record_local on cluster ck_cluster;
RENAME TABLE tsg_galaxy_v3.proxy_event_local_2402 to tsg_galaxy_v3.proxy_event_local on cluster ck_cluster;
RENAME TABLE tsg_galaxy_v3.dos_event_local_2402 to tsg_galaxy_v3.dos_event_local on cluster ck_cluster;

0
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/04_init_new_table.sql → Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_init_new_table.sql
View File

0
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/05_check.sql → Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/03_check.sql
View File

42
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/02_create_table_2307_to_2402_view.sql → Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/04_create_table_2307_to_2402_view.sql
View File

@@ -2,9 +2,9 @@ set distributed_ddl_task_timeout = 180;
-- 创建同步源表同步2402临时表物化视图, 七个表 -- 创建同步源表同步2402临时表物化视图, 七个表
-- tsg_galaxy_v3.session_record_local_2402 -- tsg_galaxy_v3.session_record_local
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.session_record_local_2307_to_2402_view on cluster ck_cluster CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.session_record_local_2307_to_2402_view on cluster ck_cluster
TO tsg_galaxy_v3.session_record_local_2402 TO tsg_galaxy_v3.session_record_local
( (
recv_time Int64, recv_time Int64,
log_id UInt64, log_id UInt64,
@@ -457,12 +457,12 @@ SELECT
common_has_dup_traffic as dup_traffic_flag, common_has_dup_traffic as dup_traffic_flag,
common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc,
common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.session_record_local FROM tsg_galaxy_v3.session_record_local_old
; ;
-- tsg_galaxy_v3.security_event_local_2402 -- tsg_galaxy_v3.security_event_local
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_local_2307_to_security_event_local_2402_view on cluster ck_cluster CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_local_2307_to_security_event_local_2402_view on cluster ck_cluster
TO tsg_galaxy_v3.security_event_local_2402 TO tsg_galaxy_v3.security_event_local
( (
recv_time Int64, recv_time Int64,
log_id UInt64, log_id UInt64,
@@ -919,13 +919,13 @@ SELECT
common_has_dup_traffic as dup_traffic_flag, common_has_dup_traffic as dup_traffic_flag,
common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc,
common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.security_event_local FROM tsg_galaxy_v3.security_event_local_old
where common_action in (16, 96) where common_action in (16, 96)
; ;
-- tsg_galaxy_v3.monitor_event_local_2402 -- tsg_galaxy_v3.monitor_event_local
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_local_2307_to_monitor_event_local_2402_view on cluster ck_cluster CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_local_2307_to_monitor_event_local_2402_view on cluster ck_cluster
TO tsg_galaxy_v3.monitor_event_local_2402 TO tsg_galaxy_v3.monitor_event_local
( (
recv_time Int64, recv_time Int64,
log_id UInt64, log_id UInt64,
@@ -1382,13 +1382,13 @@ SELECT
common_has_dup_traffic as dup_traffic_flag, common_has_dup_traffic as dup_traffic_flag,
common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc,
common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.security_event_local FROM tsg_galaxy_v3.security_event_local_old
where common_action = 1 where common_action = 1
; ;
-- tsg_galaxy_v3.transaction_record_local_2402 -- tsg_galaxy_v3.transaction_record_local
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.transaction_record_local_2307_to_2402_view on cluster ck_cluster CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.transaction_record_local_2307_to_2402_view on cluster ck_cluster
TO tsg_galaxy_v3.transaction_record_local_2402 TO tsg_galaxy_v3.transaction_record_local
( (
recv_time Int64, recv_time Int64,
log_id UInt64, log_id UInt64,
@@ -1565,12 +1565,12 @@ SELECT
sip_responder_sdp_content as sip_responder_sdp_content, sip_responder_sdp_content as sip_responder_sdp_content,
sip_duration_s as sip_duration_s, sip_duration_s as sip_duration_s,
sip_bye as sip_bye sip_bye as sip_bye
FROM tsg_galaxy_v3.transaction_record_local FROM tsg_galaxy_v3.transaction_record_local_old
; ;
-- tsg_galaxy_v3.voip_record_local_2402 -- tsg_galaxy_v3.voip_record_local
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.voip_record_local_2307_to_2402_view on cluster ck_cluster CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.voip_record_local_2307_to_2402_view on cluster ck_cluster
TO tsg_galaxy_v3.voip_record_local_2402 TO tsg_galaxy_v3.voip_record_local
( (
recv_time Int64, recv_time Int64,
log_id UInt64, log_id UInt64,
@@ -1801,12 +1801,12 @@ SELECT
common_has_dup_traffic as dup_traffic_flag, common_has_dup_traffic as dup_traffic_flag,
common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc,
common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.voip_record_local FROM tsg_galaxy_v3.voip_record_local_old
; ;
-- tsg_galaxy_v3.proxy_event_local_2402 -- tsg_galaxy_v3.proxy_event_local
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.proxy_event_local_2307_to_2402_view on cluster ck_cluster CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.proxy_event_local_2307_to_2402_view on cluster ck_cluster
TO tsg_galaxy_v3.proxy_event_local_2402 TO tsg_galaxy_v3.proxy_event_local
( (
recv_time Int64, recv_time Int64,
log_id UInt64, log_id UInt64,
@@ -2099,12 +2099,12 @@ SELECT
common_has_dup_traffic as dup_traffic_flag, common_has_dup_traffic as dup_traffic_flag,
common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc, common_tunnel_endpoint_a_desc as tunnel_endpoint_a_desc,
common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc common_tunnel_endpoint_b_desc as tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.proxy_event_local FROM tsg_galaxy_v3.proxy_event_local_old
; ;
-- tsg_galaxy_v3.dos_event_local_2402 -- tsg_galaxy_v3.dos_event_local
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.dos_event_local_2307_to_2402_view on cluster ck_cluster CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.dos_event_local_2307_to_2402_view on cluster ck_cluster
TO tsg_galaxy_v3.dos_event_local_2402 TO tsg_galaxy_v3.dos_event_local
( (
vsys_id Int32, vsys_id Int32,
recv_time Int64, recv_time Int64,
@@ -2141,5 +2141,5 @@ SELECT
session_rate as session_rate, session_rate as session_rate,
packet_rate as packet_rate, packet_rate as packet_rate,
bit_rate as bit_rate bit_rate as bit_rate
FROM tsg_galaxy_v3.dos_event_local FROM tsg_galaxy_v3.dos_event_local_old
; ;

10
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/05_drop_table_2307_to_2402_view.sql Normal file
View File

@@ -0,0 +1,10 @@
set distributed_ddl_task_timeout = 180;
-- 删除旧表同步新表物化视图, 七个表
drop view if exists tsg_galaxy_v3.session_record_local_2307_to_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.security_event_local_2307_to_security_event_local_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.security_event_local_2307_to_monitor_event_local_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.transaction_record_local_2307_to_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.voip_record_local_2307_to_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.proxy_event_local_2307_to_2402_view on cluster ck_cluster;
drop view if exists tsg_galaxy_v3.dos_event_local_2307_to_2402_view on cluster ck_cluster;

74
Clickhouse最新全量建表语句/TSG 2307版本到2402版本升级操作/ck表升级步骤.md
View File

@@ -6,50 +6,64 @@
clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1" clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1"
若返回结果为空则可执行升级步骤，否则需要等待。 若返回结果为空则可执行升级步骤，否则需要等待。
# 一、实时同步任务 # 一、停止旧表ck入库任务
* 1.创建临时表 停止旧表ck入库任务
```sh
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_create_table_2402.sql
```
* 2.创建源表同步到临时表的物化视图 # 二、旧表重命名为历史表
```sh
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 02_create_table_2307_to_2402_view.sql
```
# 二、升级各个数据中心(可选) 重命名旧表, 删除废弃表
* 1.国家中心启动ck入库任务(XX_2402 task)同步临时表：创建kafka临时topic(以_2402结尾)，启动ck同步到临时表任务
* 2.升级各个分数据中心：启动ETL任务发送到国家中心临时topic(以_2402结尾)
# 三、所有分中心升级完毕，临时表切换为目标表，源表切换为历史表
* 1.停止源表ck入库任务
* 2.停止ck入库临时表任务
* 3.重命名旧表和临时表
```sql ```sql
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 03_rename_table.sql clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 01_rename_old_table.sql
``` ```
* 4.执行2402版本初始化建表语句 # 三、初始化新表
* 1.执行2402版本初始化建表语句
``` ```
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 04_init_new_table.sql clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 02_init_new_table.sql
``` ```
* 5.校验表结构 * 2.校验表结构
``` ```
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 05_check.sql clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 03_check.sql
``` ```
无报错信息说明校验通过
* 6.启动目标表ck入库任务(升级完成) **无报错信息说明校验通过**
# 四、创建旧表同步新表任务(可选)
创建旧表同步到新表的物化视图(如果还有分数据中心向旧表写数据)
```sh
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 04_create_table_2307_to_2402_view.sql
```
# 五、启动ck入库任务
* 1.启动新表ck入库任务
* 2.启动旧表ck入库任务(如果还有分数据中心向旧表写数据)
```sh
# 重命名旧表, 删除废弃表后, 存在的旧表:
tsg_galaxy_v3.session_record_local_old
tsg_galaxy_v3.security_event_local_old
tsg_galaxy_v3.transaction_record_local_old
tsg_galaxy_v3.voip_record_local_old
tsg_galaxy_v3.proxy_event_local_old
tsg_galaxy_v3.dos_event_local_old
```
# 六、各个数据中心全部升级完成后停止旧表ck入库任务
* 1.升级各个数据中心，各个数据中心全部升级完成后，停止旧表ck入库任务(如果启动的话)
* 2.删除旧表同步新表物化视图
```sh
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < 05_drop_table_2307_to_2402_view.sql
```
# 四、离线同步历史数据(可选) # 七、离线同步历史数据(可选)
在query节点执行以下步骤，iplist.txt中为ck所有data节点ip地址。 在query节点执行以下步骤，iplist.txt中为ck所有data节点ip地址。