TSG-23742 clickhouse表结构修改，增加数据迁移脚本
This commit is contained in:
wangkuan
@@ -0,0 +1,147 @@
|
||||
assessment_event，dos_event两个表重构主键需要进行数据迁移
|
||||
具体步骤：
|
||||
|
||||
Step1 ：停止入库任务。
|
||||
Step2 ：旧版本clickhouse库表迁移至tsg_galaxy_tmp（需要确定不存在tsg_galaxy_tmp数据库）
|
||||
Step3 : tsg_galaxy_v3数据库新建重构主键库表
|
||||
Step4 : 启动入库任务->tsg_galaxy_v3
|
||||
Step5 : 迁移脚本迁移tsg_galaxy_tmp(24.11)->tsg_galaxy_v3(24.11)
|
||||
|
||||
说明
|
||||
|
||||
请按步骤依次执行,执行脚本报错时联系研发处理后再执行之后的步骤。
|
||||
所有ck步骤都需要在query节点执行
|
||||
执行所有sql语句之前需要停止日志留存调度任务,确保ck中无分布式ddl语句执行,否则执行的sql会阻塞住,影响后续步骤执行
|
||||
验证sql需要在所有query节点执行
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -u default --password ****** --query "select query from system.distributed_ddl_queue where status =0 limit 1"
|
||||
若返回结果为空则可执行升级步骤,否则需要等待。
|
||||
|
||||
|
||||
一、停止旧表ck入库任务
|
||||
停止旧表ck入库任务
|
||||
|
||||
二、旧版本clickhouse库表迁移至tsg_galaxy_tmp
|
||||
|
||||
1.查看迁移前v3库表数据量
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_v3_old_table_row_count.sql
|
||||
|
||||
|
||||
2.执行迁移sql
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < move_v3_to_tmp.sql
|
||||
|
||||
|
||||
3.查看tmp库old表数据量
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < cat_tmp_old_table_row_count.sql
|
||||
|
||||
|
||||
三、tsg_galaxy_v3数据库新建目标版本库表并进行校验
|
||||
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ****** --distributed_ddl_task_timeout 180 < update24.11.sql
|
||||
|
||||
四、启动ck入库任务
|
||||
五、离线迁移数据
|
||||
离线脚本同步历史数据至新建数据表
|
||||
在query节点执行以下步骤,iplist.txt中为ck所有data节点ip地址。
|
||||
步骤描述：
|
||||
|
||||
1.进入migrate_table_2411文件夹,使脚本可执行
|
||||
|
||||
|
||||
chmod +x ./*.sh
|
||||
|
||||
|
||||
|
||||
2.分发迁移脚本到data节点
|
||||
|
||||
|
||||
./01_send_migrate_table_scripts.sh
|
||||
|
||||
|
||||
|
||||
2.选择迁移某个表,同步需要时间区间的数据,时间区间:实时同步任务开始时间向前推n天, 实时同步任务开始时间),时间区间为左闭右开,不包含结束时间点。
|
||||
|
||||
|
||||
# 迁移assessment_event表
|
||||
./02_start_migrate_table.sh assessment_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
|
||||
|
||||
|
||||
3.监控data节点迁移情况,所有表迁移完成后,确认每个节点同步数据成功/失败批次数,如有失败批次确认是否需要处理
|
||||
|
||||
|
||||
# 监控assessment_event表迁移
|
||||
./03_monitor_migrate_table.sh assessment_event
|
||||
|
||||
|
||||
|
||||
4.选择需要迁移的表,重复2-4步骤。支持选择迁移的表有: assessment_event, dos_event。
|
||||
|
||||
迁移和监控各个表执行命令示例：
|
||||
|
||||
# 迁移assessment_event表
|
||||
./02_start_migrate_table.sh assessment_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控assessment_event表迁移
|
||||
./03_monitor_migrate_table.sh assessment_event
|
||||
|
||||
|
||||
# 迁移dos_event表
|
||||
./02_start_migrate_table.sh dos_event "2024-01-10 00:00:00" "2024-01-20 00:00:00" 60
|
||||
# 监控dos_event表迁移
|
||||
./03_monitor_migrate_table.sh dos_event
|
||||
|
||||
|
||||
迁移日志无报错,数据迁移完成。
|
||||
如果有数据迁移失败批次,查看新老表迁移数据量对应情况(ck每台data节点)：
|
||||
|
||||
|
||||
-- assessment_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.assessment_event_local_old
|
||||
WHERE recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.assessment_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
-- dos_event
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.dos_event_local_old
|
||||
WHERE recv_time>= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
SELECT
|
||||
date_trunc('day', toDateTime(recv_time)) d,
|
||||
COUNT(1) cnt
|
||||
FROM tsg_galaxy_tmp.dos_event_local
|
||||
WHERE recv_time >= toUnixTimestamp('2024-01-10 00:00:00') and recv_time < toUnixTimestamp('2024-01-20 00:00:00')
|
||||
group by date_trunc('day', toDateTime(recv_time))
|
||||
order by d
|
||||
;
|
||||
|
||||
|
||||
|
||||
六、命令迁移tsg_galaxy_tmp目标版本如（24.11）->tsg_galaxy_v3目标版本如（24.11）,按照（天）partition手动迁移
|
||||
|
||||
登录命令行之后手动按需执行需要迁移的分区（如下示例迁移2024年4月1日的数据）
|
||||
clickhouse-client -h 127.0.0.1 --port 9001 -m -n -u default --password ******
|
||||
|
||||
ALTER TABLE tsg_galaxy_tmp.assessment_event_local on cluster ck_cluster move partition 20240401 to table tsg_galaxy_v3.assessment_event_local;
|
||||
ALTER TABLE tsg_galaxy_tmp.dos_event_local on cluster ck_cluster move partition 20240401 to table tsg_galaxy_v3.dos_event_local;
|
||||
@@ -0,0 +1,3 @@
|
||||
select 'assessment_event_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.assessment_event_old;
|
||||
|
||||
select 'dos_event_old' as table_name, count(*) as cnt from tsg_galaxy_tmp.dos_event_old;
|
||||
@@ -0,0 +1,3 @@
|
||||
select 'assessment_event' as table_name, count(*) as cnt from tsg_galaxy_v3.assessment_event;
|
||||
|
||||
select 'dos_event' as table_name, count(*) as cnt from tsg_galaxy_v3.dos_event;
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
# 遍历每个节点执行迁移
|
||||
for ip in `cat iplist.txt`
|
||||
do
|
||||
# 后台执行，输出日志
|
||||
echo "$ip节点"
|
||||
ssh $ip "[ ! -d $home ] && mkdir -p $home"
|
||||
scp -r $home/*local_table_to_2411.sh $ip:$home/
|
||||
ssh $ip "cd $home && chmod +x ./*.sh"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据
|
||||
|
||||
# 迁移表 参数，可选值：dos_event，assessment_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00"
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00"
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数, 例如:240
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"assessment_event")
|
||||
# 迁移session_record
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:dos_event，assessment_event"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "slice_interval_minute参数必须是正确的分钟数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
# 遍历每个节点执行迁移
|
||||
for ip in `cat iplist.txt`
|
||||
do
|
||||
# 后台执行，输出日志
|
||||
echo "$ip 节点开始执行迁移"
|
||||
ssh $ip "cd $home && chmod +x ./*.sh && ./start_migrate_local_table_to_2411.sh $table '$data_start_time' '$data_end_time' $slice_interval_minute"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
home=$(cd `dirname $0`; pwd)
|
||||
|
||||
table=$1
|
||||
if [ -z "$table" ]; then
|
||||
echo "缺少table参数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ips=($(cat iplist.txt))
|
||||
ips_size=${#ips[*]}
|
||||
ip_starts=$( seq 0 $(($ips_size - 1)) )
|
||||
ip_ends=$( seq 0 $(($ips_size - 1)) )
|
||||
|
||||
for ((i=0;i<$ips_size;i++))
|
||||
do
|
||||
ip_starts[$i]=0
|
||||
ip_ends[$i]=0
|
||||
done
|
||||
|
||||
while true ; do
|
||||
# 遍历每个节点
|
||||
for ((i=0;i<$ips_size;i++)); do
|
||||
ip=${ips[$i]}
|
||||
start=${ip_starts[$i]}
|
||||
end=${ip_ends[$i]}
|
||||
if [ $start -eq 0 ]; then
|
||||
info=$(ssh $ip "cat $home/log_$table.txt | grep migrate_table_start")
|
||||
if [ -n "$info" ]; then
|
||||
echo "${ip}迁移开始:${info}"
|
||||
ip_starts[$i]=1
|
||||
start=1
|
||||
fi
|
||||
fi
|
||||
if [ $start -eq 1 ] && [ $end -eq 0 ] ; then
|
||||
info=$(ssh $ip "cat $home/log_$table.txt | grep migrate_table_end")
|
||||
if [ -n "$info" ]; then
|
||||
echo "${ip}迁移结束:${info}"
|
||||
ip_ends[$i]=1
|
||||
end=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
#全部结束
|
||||
finish_cnt=0
|
||||
for ((i=0;i<$ips_size;i++)); do
|
||||
start=${ip_starts[$i]}
|
||||
end=${ip_ends[$i]}
|
||||
if [ $start -eq 1 ] && [ $end -eq 1 ] ; then
|
||||
finish_cnt=$(($finish_cnt+1))
|
||||
fi
|
||||
done
|
||||
if [ $finish_cnt -ge $ips_size ]; then
|
||||
echo "所有节点迁移结束"
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
@@ -0,0 +1 @@
|
||||
192.168.44.12
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,636 @@
|
||||
#####################start 2024-11-13 07:59:56#######################
|
||||
migrate_table_start:2024-11-13 07:59:56
|
||||
输入参数: table:dos_event
|
||||
输入参数: data_start_time:2024-10-01 00:00:00(1727740800)
|
||||
输入参数: data_end_time:2024-11-22 00:00:00(1732233600)
|
||||
split time ranges:
|
||||
time_range:2024-11-21 14:00:00, 2024-11-22 00:00:00
|
||||
time_range:2024-11-21 04:00:00, 2024-11-21 14:00:00
|
||||
time_range:2024-11-20 18:00:00, 2024-11-21 04:00:00
|
||||
time_range:2024-11-20 08:00:00, 2024-11-20 18:00:00
|
||||
time_range:2024-11-19 22:00:00, 2024-11-20 08:00:00
|
||||
time_range:2024-11-19 12:00:00, 2024-11-19 22:00:00
|
||||
time_range:2024-11-19 02:00:00, 2024-11-19 12:00:00
|
||||
time_range:2024-11-18 16:00:00, 2024-11-19 02:00:00
|
||||
time_range:2024-11-18 06:00:00, 2024-11-18 16:00:00
|
||||
time_range:2024-11-17 20:00:00, 2024-11-18 06:00:00
|
||||
time_range:2024-11-17 10:00:00, 2024-11-17 20:00:00
|
||||
time_range:2024-11-17 00:00:00, 2024-11-17 10:00:00
|
||||
time_range:2024-11-16 14:00:00, 2024-11-17 00:00:00
|
||||
time_range:2024-11-16 04:00:00, 2024-11-16 14:00:00
|
||||
time_range:2024-11-15 18:00:00, 2024-11-16 04:00:00
|
||||
time_range:2024-11-15 08:00:00, 2024-11-15 18:00:00
|
||||
time_range:2024-11-14 22:00:00, 2024-11-15 08:00:00
|
||||
time_range:2024-11-14 12:00:00, 2024-11-14 22:00:00
|
||||
time_range:2024-11-14 02:00:00, 2024-11-14 12:00:00
|
||||
time_range:2024-11-13 16:00:00, 2024-11-14 02:00:00
|
||||
time_range:2024-11-13 06:00:00, 2024-11-13 16:00:00
|
||||
time_range:2024-11-12 20:00:00, 2024-11-13 06:00:00
|
||||
time_range:2024-11-12 10:00:00, 2024-11-12 20:00:00
|
||||
time_range:2024-11-12 00:00:00, 2024-11-12 10:00:00
|
||||
time_range:2024-11-11 14:00:00, 2024-11-12 00:00:00
|
||||
time_range:2024-11-11 04:00:00, 2024-11-11 14:00:00
|
||||
time_range:2024-11-10 18:00:00, 2024-11-11 04:00:00
|
||||
time_range:2024-11-10 08:00:00, 2024-11-10 18:00:00
|
||||
time_range:2024-11-09 22:00:00, 2024-11-10 08:00:00
|
||||
time_range:2024-11-09 12:00:00, 2024-11-09 22:00:00
|
||||
time_range:2024-11-09 02:00:00, 2024-11-09 12:00:00
|
||||
time_range:2024-11-08 16:00:00, 2024-11-09 02:00:00
|
||||
time_range:2024-11-08 06:00:00, 2024-11-08 16:00:00
|
||||
time_range:2024-11-07 20:00:00, 2024-11-08 06:00:00
|
||||
time_range:2024-11-07 10:00:00, 2024-11-07 20:00:00
|
||||
time_range:2024-11-07 00:00:00, 2024-11-07 10:00:00
|
||||
time_range:2024-11-06 14:00:00, 2024-11-07 00:00:00
|
||||
time_range:2024-11-06 04:00:00, 2024-11-06 14:00:00
|
||||
time_range:2024-11-05 18:00:00, 2024-11-06 04:00:00
|
||||
time_range:2024-11-05 08:00:00, 2024-11-05 18:00:00
|
||||
time_range:2024-11-04 22:00:00, 2024-11-05 08:00:00
|
||||
time_range:2024-11-04 12:00:00, 2024-11-04 22:00:00
|
||||
time_range:2024-11-04 02:00:00, 2024-11-04 12:00:00
|
||||
time_range:2024-11-03 16:00:00, 2024-11-04 02:00:00
|
||||
time_range:2024-11-03 06:00:00, 2024-11-03 16:00:00
|
||||
time_range:2024-11-02 20:00:00, 2024-11-03 06:00:00
|
||||
time_range:2024-11-02 10:00:00, 2024-11-02 20:00:00
|
||||
time_range:2024-11-02 00:00:00, 2024-11-02 10:00:00
|
||||
time_range:2024-11-01 14:00:00, 2024-11-02 00:00:00
|
||||
time_range:2024-11-01 04:00:00, 2024-11-01 14:00:00
|
||||
time_range:2024-10-31 18:00:00, 2024-11-01 04:00:00
|
||||
time_range:2024-10-31 08:00:00, 2024-10-31 18:00:00
|
||||
time_range:2024-10-30 22:00:00, 2024-10-31 08:00:00
|
||||
time_range:2024-10-30 12:00:00, 2024-10-30 22:00:00
|
||||
time_range:2024-10-30 02:00:00, 2024-10-30 12:00:00
|
||||
time_range:2024-10-29 16:00:00, 2024-10-30 02:00:00
|
||||
time_range:2024-10-29 06:00:00, 2024-10-29 16:00:00
|
||||
time_range:2024-10-28 20:00:00, 2024-10-29 06:00:00
|
||||
time_range:2024-10-28 10:00:00, 2024-10-28 20:00:00
|
||||
time_range:2024-10-28 00:00:00, 2024-10-28 10:00:00
|
||||
time_range:2024-10-27 14:00:00, 2024-10-28 00:00:00
|
||||
time_range:2024-10-27 04:00:00, 2024-10-27 14:00:00
|
||||
time_range:2024-10-26 18:00:00, 2024-10-27 04:00:00
|
||||
time_range:2024-10-26 08:00:00, 2024-10-26 18:00:00
|
||||
time_range:2024-10-25 22:00:00, 2024-10-26 08:00:00
|
||||
time_range:2024-10-25 12:00:00, 2024-10-25 22:00:00
|
||||
time_range:2024-10-25 02:00:00, 2024-10-25 12:00:00
|
||||
time_range:2024-10-24 16:00:00, 2024-10-25 02:00:00
|
||||
time_range:2024-10-24 06:00:00, 2024-10-24 16:00:00
|
||||
time_range:2024-10-23 20:00:00, 2024-10-24 06:00:00
|
||||
time_range:2024-10-23 10:00:00, 2024-10-23 20:00:00
|
||||
time_range:2024-10-23 00:00:00, 2024-10-23 10:00:00
|
||||
time_range:2024-10-22 14:00:00, 2024-10-23 00:00:00
|
||||
time_range:2024-10-22 04:00:00, 2024-10-22 14:00:00
|
||||
time_range:2024-10-21 18:00:00, 2024-10-22 04:00:00
|
||||
time_range:2024-10-21 08:00:00, 2024-10-21 18:00:00
|
||||
time_range:2024-10-20 22:00:00, 2024-10-21 08:00:00
|
||||
time_range:2024-10-20 12:00:00, 2024-10-20 22:00:00
|
||||
time_range:2024-10-20 02:00:00, 2024-10-20 12:00:00
|
||||
time_range:2024-10-19 16:00:00, 2024-10-20 02:00:00
|
||||
time_range:2024-10-19 06:00:00, 2024-10-19 16:00:00
|
||||
time_range:2024-10-18 20:00:00, 2024-10-19 06:00:00
|
||||
time_range:2024-10-18 10:00:00, 2024-10-18 20:00:00
|
||||
time_range:2024-10-18 00:00:00, 2024-10-18 10:00:00
|
||||
time_range:2024-10-17 14:00:00, 2024-10-18 00:00:00
|
||||
time_range:2024-10-17 04:00:00, 2024-10-17 14:00:00
|
||||
time_range:2024-10-16 18:00:00, 2024-10-17 04:00:00
|
||||
time_range:2024-10-16 08:00:00, 2024-10-16 18:00:00
|
||||
time_range:2024-10-15 22:00:00, 2024-10-16 08:00:00
|
||||
time_range:2024-10-15 12:00:00, 2024-10-15 22:00:00
|
||||
time_range:2024-10-15 02:00:00, 2024-10-15 12:00:00
|
||||
time_range:2024-10-14 16:00:00, 2024-10-15 02:00:00
|
||||
time_range:2024-10-14 06:00:00, 2024-10-14 16:00:00
|
||||
time_range:2024-10-13 20:00:00, 2024-10-14 06:00:00
|
||||
time_range:2024-10-13 10:00:00, 2024-10-13 20:00:00
|
||||
time_range:2024-10-13 00:00:00, 2024-10-13 10:00:00
|
||||
time_range:2024-10-12 14:00:00, 2024-10-13 00:00:00
|
||||
time_range:2024-10-12 04:00:00, 2024-10-12 14:00:00
|
||||
time_range:2024-10-11 18:00:00, 2024-10-12 04:00:00
|
||||
time_range:2024-10-11 08:00:00, 2024-10-11 18:00:00
|
||||
time_range:2024-10-10 22:00:00, 2024-10-11 08:00:00
|
||||
time_range:2024-10-10 12:00:00, 2024-10-10 22:00:00
|
||||
time_range:2024-10-10 02:00:00, 2024-10-10 12:00:00
|
||||
time_range:2024-10-09 16:00:00, 2024-10-10 02:00:00
|
||||
time_range:2024-10-09 06:00:00, 2024-10-09 16:00:00
|
||||
time_range:2024-10-08 20:00:00, 2024-10-09 06:00:00
|
||||
time_range:2024-10-08 10:00:00, 2024-10-08 20:00:00
|
||||
time_range:2024-10-08 00:00:00, 2024-10-08 10:00:00
|
||||
time_range:2024-10-07 14:00:00, 2024-10-08 00:00:00
|
||||
time_range:2024-10-07 04:00:00, 2024-10-07 14:00:00
|
||||
time_range:2024-10-06 18:00:00, 2024-10-07 04:00:00
|
||||
time_range:2024-10-06 08:00:00, 2024-10-06 18:00:00
|
||||
time_range:2024-10-05 22:00:00, 2024-10-06 08:00:00
|
||||
time_range:2024-10-05 12:00:00, 2024-10-05 22:00:00
|
||||
time_range:2024-10-05 02:00:00, 2024-10-05 12:00:00
|
||||
time_range:2024-10-04 16:00:00, 2024-10-05 02:00:00
|
||||
time_range:2024-10-04 06:00:00, 2024-10-04 16:00:00
|
||||
time_range:2024-10-03 20:00:00, 2024-10-04 06:00:00
|
||||
time_range:2024-10-03 10:00:00, 2024-10-03 20:00:00
|
||||
time_range:2024-10-03 00:00:00, 2024-10-03 10:00:00
|
||||
time_range:2024-10-02 14:00:00, 2024-10-03 00:00:00
|
||||
time_range:2024-10-02 04:00:00, 2024-10-02 14:00:00
|
||||
time_range:2024-10-01 18:00:00, 2024-10-02 04:00:00
|
||||
time_range:2024-10-01 08:00:00, 2024-10-01 18:00:00
|
||||
time_range:2024-10-01 00:00:00, 2024-10-01 08:00:00
|
||||
**********start migrate dos_event table in 2024-11-13 07:59:56**********
|
||||
start dos_event part for time_range:2024-11-21 14:00:00(1732197600), 2024-11-22 00:00:00(1732233600), in 2024-11-13 07:59:56
|
||||
同步数据段成功:[2024-11-21 14:00:00, 2024-11-22 00:00:00]. in 2024-11-13 07:59:56
|
||||
end dos_event part for time_range:2024-11-21 14:00:00(1732197600), 2024-11-22 00:00:00(1732233600), in 2024-11-13 07:59:56
|
||||
|
||||
start dos_event part for time_range:2024-11-21 04:00:00(1732161600), 2024-11-21 14:00:00(1732197600), in 2024-11-13 07:59:56
|
||||
同步数据段成功:[2024-11-21 04:00:00, 2024-11-21 14:00:00]. in 2024-11-13 07:59:56
|
||||
end dos_event part for time_range:2024-11-21 04:00:00(1732161600), 2024-11-21 14:00:00(1732197600), in 2024-11-13 07:59:56
|
||||
|
||||
start dos_event part for time_range:2024-11-20 18:00:00(1732125600), 2024-11-21 04:00:00(1732161600), in 2024-11-13 07:59:56
|
||||
同步数据段成功:[2024-11-20 18:00:00, 2024-11-21 04:00:00]. in 2024-11-13 07:59:56
|
||||
end dos_event part for time_range:2024-11-20 18:00:00(1732125600), 2024-11-21 04:00:00(1732161600), in 2024-11-13 07:59:56
|
||||
|
||||
start dos_event part for time_range:2024-11-20 08:00:00(1732089600), 2024-11-20 18:00:00(1732125600), in 2024-11-13 07:59:56
|
||||
同步数据段成功:[2024-11-20 08:00:00, 2024-11-20 18:00:00]. in 2024-11-13 07:59:56
|
||||
end dos_event part for time_range:2024-11-20 08:00:00(1732089600), 2024-11-20 18:00:00(1732125600), in 2024-11-13 07:59:56
|
||||
|
||||
start dos_event part for time_range:2024-11-19 22:00:00(1732053600), 2024-11-20 08:00:00(1732089600), in 2024-11-13 07:59:56
|
||||
同步数据段成功:[2024-11-19 22:00:00, 2024-11-20 08:00:00]. in 2024-11-13 07:59:56
|
||||
end dos_event part for time_range:2024-11-19 22:00:00(1732053600), 2024-11-20 08:00:00(1732089600), in 2024-11-13 07:59:56
|
||||
|
||||
start dos_event part for time_range:2024-11-19 12:00:00(1732017600), 2024-11-19 22:00:00(1732053600), in 2024-11-13 07:59:56
|
||||
同步数据段成功:[2024-11-19 12:00:00, 2024-11-19 22:00:00]. in 2024-11-13 07:59:56
|
||||
end dos_event part for time_range:2024-11-19 12:00:00(1732017600), 2024-11-19 22:00:00(1732053600), in 2024-11-13 07:59:56
|
||||
|
||||
start dos_event part for time_range:2024-11-19 02:00:00(1731981600), 2024-11-19 12:00:00(1732017600), in 2024-11-13 07:59:56
|
||||
同步数据段成功:[2024-11-19 02:00:00, 2024-11-19 12:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-19 02:00:00(1731981600), 2024-11-19 12:00:00(1732017600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-18 16:00:00(1731945600), 2024-11-19 02:00:00(1731981600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-18 16:00:00, 2024-11-19 02:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-18 16:00:00(1731945600), 2024-11-19 02:00:00(1731981600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-18 06:00:00(1731909600), 2024-11-18 16:00:00(1731945600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-18 06:00:00, 2024-11-18 16:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-18 06:00:00(1731909600), 2024-11-18 16:00:00(1731945600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-17 20:00:00(1731873600), 2024-11-18 06:00:00(1731909600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-17 20:00:00, 2024-11-18 06:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-17 20:00:00(1731873600), 2024-11-18 06:00:00(1731909600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-17 10:00:00(1731837600), 2024-11-17 20:00:00(1731873600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-17 10:00:00, 2024-11-17 20:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-17 10:00:00(1731837600), 2024-11-17 20:00:00(1731873600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-17 00:00:00(1731801600), 2024-11-17 10:00:00(1731837600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-17 00:00:00, 2024-11-17 10:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-17 00:00:00(1731801600), 2024-11-17 10:00:00(1731837600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-16 14:00:00(1731765600), 2024-11-17 00:00:00(1731801600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-16 14:00:00, 2024-11-17 00:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-16 14:00:00(1731765600), 2024-11-17 00:00:00(1731801600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-16 04:00:00(1731729600), 2024-11-16 14:00:00(1731765600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-16 04:00:00, 2024-11-16 14:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-16 04:00:00(1731729600), 2024-11-16 14:00:00(1731765600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-15 18:00:00(1731693600), 2024-11-16 04:00:00(1731729600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-15 18:00:00, 2024-11-16 04:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-15 18:00:00(1731693600), 2024-11-16 04:00:00(1731729600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-15 08:00:00(1731657600), 2024-11-15 18:00:00(1731693600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-15 08:00:00, 2024-11-15 18:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-15 08:00:00(1731657600), 2024-11-15 18:00:00(1731693600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-14 22:00:00(1731621600), 2024-11-15 08:00:00(1731657600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-14 22:00:00, 2024-11-15 08:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-14 22:00:00(1731621600), 2024-11-15 08:00:00(1731657600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-14 12:00:00(1731585600), 2024-11-14 22:00:00(1731621600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-14 12:00:00, 2024-11-14 22:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-14 12:00:00(1731585600), 2024-11-14 22:00:00(1731621600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-14 02:00:00(1731549600), 2024-11-14 12:00:00(1731585600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-14 02:00:00, 2024-11-14 12:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-14 02:00:00(1731549600), 2024-11-14 12:00:00(1731585600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-13 16:00:00(1731513600), 2024-11-14 02:00:00(1731549600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-13 16:00:00, 2024-11-14 02:00:00]. in 2024-11-13 07:59:57
|
||||
end dos_event part for time_range:2024-11-13 16:00:00(1731513600), 2024-11-14 02:00:00(1731549600), in 2024-11-13 07:59:57
|
||||
|
||||
start dos_event part for time_range:2024-11-13 06:00:00(1731477600), 2024-11-13 16:00:00(1731513600), in 2024-11-13 07:59:57
|
||||
同步数据段成功:[2024-11-13 06:00:00, 2024-11-13 16:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-13 06:00:00(1731477600), 2024-11-13 16:00:00(1731513600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-12 20:00:00(1731441600), 2024-11-13 06:00:00(1731477600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-12 20:00:00, 2024-11-13 06:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-12 20:00:00(1731441600), 2024-11-13 06:00:00(1731477600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-12 10:00:00(1731405600), 2024-11-12 20:00:00(1731441600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-12 10:00:00, 2024-11-12 20:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-12 10:00:00(1731405600), 2024-11-12 20:00:00(1731441600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-12 00:00:00(1731369600), 2024-11-12 10:00:00(1731405600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-12 00:00:00, 2024-11-12 10:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-12 00:00:00(1731369600), 2024-11-12 10:00:00(1731405600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-11 14:00:00(1731333600), 2024-11-12 00:00:00(1731369600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-11 14:00:00, 2024-11-12 00:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-11 14:00:00(1731333600), 2024-11-12 00:00:00(1731369600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-11 04:00:00(1731297600), 2024-11-11 14:00:00(1731333600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-11 04:00:00, 2024-11-11 14:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-11 04:00:00(1731297600), 2024-11-11 14:00:00(1731333600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-10 18:00:00(1731261600), 2024-11-11 04:00:00(1731297600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-10 18:00:00, 2024-11-11 04:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-10 18:00:00(1731261600), 2024-11-11 04:00:00(1731297600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-10 08:00:00(1731225600), 2024-11-10 18:00:00(1731261600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-10 08:00:00, 2024-11-10 18:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-10 08:00:00(1731225600), 2024-11-10 18:00:00(1731261600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-09 22:00:00(1731189600), 2024-11-10 08:00:00(1731225600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-09 22:00:00, 2024-11-10 08:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-09 22:00:00(1731189600), 2024-11-10 08:00:00(1731225600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-09 12:00:00(1731153600), 2024-11-09 22:00:00(1731189600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-09 12:00:00, 2024-11-09 22:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-09 12:00:00(1731153600), 2024-11-09 22:00:00(1731189600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-09 02:00:00(1731117600), 2024-11-09 12:00:00(1731153600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-09 02:00:00, 2024-11-09 12:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-09 02:00:00(1731117600), 2024-11-09 12:00:00(1731153600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-08 16:00:00(1731081600), 2024-11-09 02:00:00(1731117600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-08 16:00:00, 2024-11-09 02:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-08 16:00:00(1731081600), 2024-11-09 02:00:00(1731117600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-08 06:00:00(1731045600), 2024-11-08 16:00:00(1731081600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-08 06:00:00, 2024-11-08 16:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-08 06:00:00(1731045600), 2024-11-08 16:00:00(1731081600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-07 20:00:00(1731009600), 2024-11-08 06:00:00(1731045600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-07 20:00:00, 2024-11-08 06:00:00]. in 2024-11-13 07:59:58
|
||||
end dos_event part for time_range:2024-11-07 20:00:00(1731009600), 2024-11-08 06:00:00(1731045600), in 2024-11-13 07:59:58
|
||||
|
||||
start dos_event part for time_range:2024-11-07 10:00:00(1730973600), 2024-11-07 20:00:00(1731009600), in 2024-11-13 07:59:58
|
||||
同步数据段成功:[2024-11-07 10:00:00, 2024-11-07 20:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-07 10:00:00(1730973600), 2024-11-07 20:00:00(1731009600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-07 00:00:00(1730937600), 2024-11-07 10:00:00(1730973600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-07 00:00:00, 2024-11-07 10:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-07 00:00:00(1730937600), 2024-11-07 10:00:00(1730973600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-06 14:00:00(1730901600), 2024-11-07 00:00:00(1730937600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-06 14:00:00, 2024-11-07 00:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-06 14:00:00(1730901600), 2024-11-07 00:00:00(1730937600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-06 04:00:00(1730865600), 2024-11-06 14:00:00(1730901600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-06 04:00:00, 2024-11-06 14:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-06 04:00:00(1730865600), 2024-11-06 14:00:00(1730901600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-05 18:00:00(1730829600), 2024-11-06 04:00:00(1730865600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-05 18:00:00, 2024-11-06 04:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-05 18:00:00(1730829600), 2024-11-06 04:00:00(1730865600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-05 08:00:00(1730793600), 2024-11-05 18:00:00(1730829600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-05 08:00:00, 2024-11-05 18:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-05 08:00:00(1730793600), 2024-11-05 18:00:00(1730829600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-04 22:00:00(1730757600), 2024-11-05 08:00:00(1730793600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-04 22:00:00, 2024-11-05 08:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-04 22:00:00(1730757600), 2024-11-05 08:00:00(1730793600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-04 12:00:00(1730721600), 2024-11-04 22:00:00(1730757600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-04 12:00:00, 2024-11-04 22:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-04 12:00:00(1730721600), 2024-11-04 22:00:00(1730757600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-04 02:00:00(1730685600), 2024-11-04 12:00:00(1730721600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-04 02:00:00, 2024-11-04 12:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-04 02:00:00(1730685600), 2024-11-04 12:00:00(1730721600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-03 16:00:00(1730649600), 2024-11-04 02:00:00(1730685600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-03 16:00:00, 2024-11-04 02:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-03 16:00:00(1730649600), 2024-11-04 02:00:00(1730685600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-03 06:00:00(1730613600), 2024-11-03 16:00:00(1730649600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-03 06:00:00, 2024-11-03 16:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-03 06:00:00(1730613600), 2024-11-03 16:00:00(1730649600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-02 20:00:00(1730577600), 2024-11-03 06:00:00(1730613600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-02 20:00:00, 2024-11-03 06:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-02 20:00:00(1730577600), 2024-11-03 06:00:00(1730613600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-02 10:00:00(1730541600), 2024-11-02 20:00:00(1730577600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-02 10:00:00, 2024-11-02 20:00:00]. in 2024-11-13 07:59:59
|
||||
end dos_event part for time_range:2024-11-02 10:00:00(1730541600), 2024-11-02 20:00:00(1730577600), in 2024-11-13 07:59:59
|
||||
|
||||
start dos_event part for time_range:2024-11-02 00:00:00(1730505600), 2024-11-02 10:00:00(1730541600), in 2024-11-13 07:59:59
|
||||
同步数据段成功:[2024-11-02 00:00:00, 2024-11-02 10:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-11-02 00:00:00(1730505600), 2024-11-02 10:00:00(1730541600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-11-01 14:00:00(1730469600), 2024-11-02 00:00:00(1730505600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-11-01 14:00:00, 2024-11-02 00:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-11-01 14:00:00(1730469600), 2024-11-02 00:00:00(1730505600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-11-01 04:00:00(1730433600), 2024-11-01 14:00:00(1730469600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-11-01 04:00:00, 2024-11-01 14:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-11-01 04:00:00(1730433600), 2024-11-01 14:00:00(1730469600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-31 18:00:00(1730397600), 2024-11-01 04:00:00(1730433600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-31 18:00:00, 2024-11-01 04:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-10-31 18:00:00(1730397600), 2024-11-01 04:00:00(1730433600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-31 08:00:00(1730361600), 2024-10-31 18:00:00(1730397600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-31 08:00:00, 2024-10-31 18:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-10-31 08:00:00(1730361600), 2024-10-31 18:00:00(1730397600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-30 22:00:00(1730325600), 2024-10-31 08:00:00(1730361600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-30 22:00:00, 2024-10-31 08:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-10-30 22:00:00(1730325600), 2024-10-31 08:00:00(1730361600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-30 12:00:00(1730289600), 2024-10-30 22:00:00(1730325600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-30 12:00:00, 2024-10-30 22:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-10-30 12:00:00(1730289600), 2024-10-30 22:00:00(1730325600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-30 02:00:00(1730253600), 2024-10-30 12:00:00(1730289600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-30 02:00:00, 2024-10-30 12:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-10-30 02:00:00(1730253600), 2024-10-30 12:00:00(1730289600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-29 16:00:00(1730217600), 2024-10-30 02:00:00(1730253600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-29 16:00:00, 2024-10-30 02:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-10-29 16:00:00(1730217600), 2024-10-30 02:00:00(1730253600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-29 06:00:00(1730181600), 2024-10-29 16:00:00(1730217600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-29 06:00:00, 2024-10-29 16:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-10-29 06:00:00(1730181600), 2024-10-29 16:00:00(1730217600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-28 20:00:00(1730145600), 2024-10-29 06:00:00(1730181600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-28 20:00:00, 2024-10-29 06:00:00]. in 2024-11-13 08:00:00
|
||||
end dos_event part for time_range:2024-10-28 20:00:00(1730145600), 2024-10-29 06:00:00(1730181600), in 2024-11-13 08:00:00
|
||||
|
||||
start dos_event part for time_range:2024-10-28 10:00:00(1730109600), 2024-10-28 20:00:00(1730145600), in 2024-11-13 08:00:00
|
||||
同步数据段成功:[2024-10-28 10:00:00, 2024-10-28 20:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-28 10:00:00(1730109600), 2024-10-28 20:00:00(1730145600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-28 00:00:00(1730073600), 2024-10-28 10:00:00(1730109600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-28 00:00:00, 2024-10-28 10:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-28 00:00:00(1730073600), 2024-10-28 10:00:00(1730109600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-27 14:00:00(1730037600), 2024-10-28 00:00:00(1730073600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-27 14:00:00, 2024-10-28 00:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-27 14:00:00(1730037600), 2024-10-28 00:00:00(1730073600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-27 04:00:00(1730001600), 2024-10-27 14:00:00(1730037600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-27 04:00:00, 2024-10-27 14:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-27 04:00:00(1730001600), 2024-10-27 14:00:00(1730037600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-26 18:00:00(1729965600), 2024-10-27 04:00:00(1730001600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-26 18:00:00, 2024-10-27 04:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-26 18:00:00(1729965600), 2024-10-27 04:00:00(1730001600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-26 08:00:00(1729929600), 2024-10-26 18:00:00(1729965600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-26 08:00:00, 2024-10-26 18:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-26 08:00:00(1729929600), 2024-10-26 18:00:00(1729965600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-25 22:00:00(1729893600), 2024-10-26 08:00:00(1729929600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-25 22:00:00, 2024-10-26 08:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-25 22:00:00(1729893600), 2024-10-26 08:00:00(1729929600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-25 12:00:00(1729857600), 2024-10-25 22:00:00(1729893600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-25 12:00:00, 2024-10-25 22:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-25 12:00:00(1729857600), 2024-10-25 22:00:00(1729893600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-25 02:00:00(1729821600), 2024-10-25 12:00:00(1729857600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-25 02:00:00, 2024-10-25 12:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-25 02:00:00(1729821600), 2024-10-25 12:00:00(1729857600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-24 16:00:00(1729785600), 2024-10-25 02:00:00(1729821600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-24 16:00:00, 2024-10-25 02:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-24 16:00:00(1729785600), 2024-10-25 02:00:00(1729821600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-24 06:00:00(1729749600), 2024-10-24 16:00:00(1729785600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-24 06:00:00, 2024-10-24 16:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-24 06:00:00(1729749600), 2024-10-24 16:00:00(1729785600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-23 20:00:00(1729713600), 2024-10-24 06:00:00(1729749600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-23 20:00:00, 2024-10-24 06:00:00]. in 2024-11-13 08:00:01
|
||||
end dos_event part for time_range:2024-10-23 20:00:00(1729713600), 2024-10-24 06:00:00(1729749600), in 2024-11-13 08:00:01
|
||||
|
||||
start dos_event part for time_range:2024-10-23 10:00:00(1729677600), 2024-10-23 20:00:00(1729713600), in 2024-11-13 08:00:01
|
||||
同步数据段成功:[2024-10-23 10:00:00, 2024-10-23 20:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-23 10:00:00(1729677600), 2024-10-23 20:00:00(1729713600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-23 00:00:00(1729641600), 2024-10-23 10:00:00(1729677600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-23 00:00:00, 2024-10-23 10:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-23 00:00:00(1729641600), 2024-10-23 10:00:00(1729677600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-22 14:00:00(1729605600), 2024-10-23 00:00:00(1729641600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-22 14:00:00, 2024-10-23 00:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-22 14:00:00(1729605600), 2024-10-23 00:00:00(1729641600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-22 04:00:00(1729569600), 2024-10-22 14:00:00(1729605600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-22 04:00:00, 2024-10-22 14:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-22 04:00:00(1729569600), 2024-10-22 14:00:00(1729605600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-21 18:00:00(1729533600), 2024-10-22 04:00:00(1729569600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-21 18:00:00, 2024-10-22 04:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-21 18:00:00(1729533600), 2024-10-22 04:00:00(1729569600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-21 08:00:00(1729497600), 2024-10-21 18:00:00(1729533600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-21 08:00:00, 2024-10-21 18:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-21 08:00:00(1729497600), 2024-10-21 18:00:00(1729533600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-20 22:00:00(1729461600), 2024-10-21 08:00:00(1729497600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-20 22:00:00, 2024-10-21 08:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-20 22:00:00(1729461600), 2024-10-21 08:00:00(1729497600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-20 12:00:00(1729425600), 2024-10-20 22:00:00(1729461600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-20 12:00:00, 2024-10-20 22:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-20 12:00:00(1729425600), 2024-10-20 22:00:00(1729461600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-20 02:00:00(1729389600), 2024-10-20 12:00:00(1729425600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-20 02:00:00, 2024-10-20 12:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-20 02:00:00(1729389600), 2024-10-20 12:00:00(1729425600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-19 16:00:00(1729353600), 2024-10-20 02:00:00(1729389600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-19 16:00:00, 2024-10-20 02:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-19 16:00:00(1729353600), 2024-10-20 02:00:00(1729389600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-19 06:00:00(1729317600), 2024-10-19 16:00:00(1729353600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-19 06:00:00, 2024-10-19 16:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-19 06:00:00(1729317600), 2024-10-19 16:00:00(1729353600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-18 20:00:00(1729281600), 2024-10-19 06:00:00(1729317600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-18 20:00:00, 2024-10-19 06:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-18 20:00:00(1729281600), 2024-10-19 06:00:00(1729317600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-18 10:00:00(1729245600), 2024-10-18 20:00:00(1729281600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-18 10:00:00, 2024-10-18 20:00:00]. in 2024-11-13 08:00:02
|
||||
end dos_event part for time_range:2024-10-18 10:00:00(1729245600), 2024-10-18 20:00:00(1729281600), in 2024-11-13 08:00:02
|
||||
|
||||
start dos_event part for time_range:2024-10-18 00:00:00(1729209600), 2024-10-18 10:00:00(1729245600), in 2024-11-13 08:00:02
|
||||
同步数据段成功:[2024-10-18 00:00:00, 2024-10-18 10:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-18 00:00:00(1729209600), 2024-10-18 10:00:00(1729245600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-17 14:00:00(1729173600), 2024-10-18 00:00:00(1729209600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-17 14:00:00, 2024-10-18 00:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-17 14:00:00(1729173600), 2024-10-18 00:00:00(1729209600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-17 04:00:00(1729137600), 2024-10-17 14:00:00(1729173600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-17 04:00:00, 2024-10-17 14:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-17 04:00:00(1729137600), 2024-10-17 14:00:00(1729173600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-16 18:00:00(1729101600), 2024-10-17 04:00:00(1729137600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-16 18:00:00, 2024-10-17 04:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-16 18:00:00(1729101600), 2024-10-17 04:00:00(1729137600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-16 08:00:00(1729065600), 2024-10-16 18:00:00(1729101600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-16 08:00:00, 2024-10-16 18:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-16 08:00:00(1729065600), 2024-10-16 18:00:00(1729101600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-15 22:00:00(1729029600), 2024-10-16 08:00:00(1729065600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-15 22:00:00, 2024-10-16 08:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-15 22:00:00(1729029600), 2024-10-16 08:00:00(1729065600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-15 12:00:00(1728993600), 2024-10-15 22:00:00(1729029600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-15 12:00:00, 2024-10-15 22:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-15 12:00:00(1728993600), 2024-10-15 22:00:00(1729029600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-15 02:00:00(1728957600), 2024-10-15 12:00:00(1728993600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-15 02:00:00, 2024-10-15 12:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-15 02:00:00(1728957600), 2024-10-15 12:00:00(1728993600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-14 16:00:00(1728921600), 2024-10-15 02:00:00(1728957600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-14 16:00:00, 2024-10-15 02:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-14 16:00:00(1728921600), 2024-10-15 02:00:00(1728957600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-14 06:00:00(1728885600), 2024-10-14 16:00:00(1728921600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-14 06:00:00, 2024-10-14 16:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-14 06:00:00(1728885600), 2024-10-14 16:00:00(1728921600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-13 20:00:00(1728849600), 2024-10-14 06:00:00(1728885600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-13 20:00:00, 2024-10-14 06:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-13 20:00:00(1728849600), 2024-10-14 06:00:00(1728885600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-13 10:00:00(1728813600), 2024-10-13 20:00:00(1728849600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-13 10:00:00, 2024-10-13 20:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-13 10:00:00(1728813600), 2024-10-13 20:00:00(1728849600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-13 00:00:00(1728777600), 2024-10-13 10:00:00(1728813600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-13 00:00:00, 2024-10-13 10:00:00]. in 2024-11-13 08:00:03
|
||||
end dos_event part for time_range:2024-10-13 00:00:00(1728777600), 2024-10-13 10:00:00(1728813600), in 2024-11-13 08:00:03
|
||||
|
||||
start dos_event part for time_range:2024-10-12 14:00:00(1728741600), 2024-10-13 00:00:00(1728777600), in 2024-11-13 08:00:03
|
||||
同步数据段成功:[2024-10-12 14:00:00, 2024-10-13 00:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-12 14:00:00(1728741600), 2024-10-13 00:00:00(1728777600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-12 04:00:00(1728705600), 2024-10-12 14:00:00(1728741600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-12 04:00:00, 2024-10-12 14:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-12 04:00:00(1728705600), 2024-10-12 14:00:00(1728741600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-11 18:00:00(1728669600), 2024-10-12 04:00:00(1728705600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-11 18:00:00, 2024-10-12 04:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-11 18:00:00(1728669600), 2024-10-12 04:00:00(1728705600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-11 08:00:00(1728633600), 2024-10-11 18:00:00(1728669600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-11 08:00:00, 2024-10-11 18:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-11 08:00:00(1728633600), 2024-10-11 18:00:00(1728669600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-10 22:00:00(1728597600), 2024-10-11 08:00:00(1728633600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-10 22:00:00, 2024-10-11 08:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-10 22:00:00(1728597600), 2024-10-11 08:00:00(1728633600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-10 12:00:00(1728561600), 2024-10-10 22:00:00(1728597600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-10 12:00:00, 2024-10-10 22:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-10 12:00:00(1728561600), 2024-10-10 22:00:00(1728597600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-10 02:00:00(1728525600), 2024-10-10 12:00:00(1728561600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-10 02:00:00, 2024-10-10 12:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-10 02:00:00(1728525600), 2024-10-10 12:00:00(1728561600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-09 16:00:00(1728489600), 2024-10-10 02:00:00(1728525600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-09 16:00:00, 2024-10-10 02:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-09 16:00:00(1728489600), 2024-10-10 02:00:00(1728525600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-09 06:00:00(1728453600), 2024-10-09 16:00:00(1728489600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-09 06:00:00, 2024-10-09 16:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-09 06:00:00(1728453600), 2024-10-09 16:00:00(1728489600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-08 20:00:00(1728417600), 2024-10-09 06:00:00(1728453600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-08 20:00:00, 2024-10-09 06:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-08 20:00:00(1728417600), 2024-10-09 06:00:00(1728453600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-08 10:00:00(1728381600), 2024-10-08 20:00:00(1728417600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-08 10:00:00, 2024-10-08 20:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-08 10:00:00(1728381600), 2024-10-08 20:00:00(1728417600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-08 00:00:00(1728345600), 2024-10-08 10:00:00(1728381600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-08 00:00:00, 2024-10-08 10:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-08 00:00:00(1728345600), 2024-10-08 10:00:00(1728381600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-07 14:00:00(1728309600), 2024-10-08 00:00:00(1728345600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-07 14:00:00, 2024-10-08 00:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-07 14:00:00(1728309600), 2024-10-08 00:00:00(1728345600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-07 04:00:00(1728273600), 2024-10-07 14:00:00(1728309600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-07 04:00:00, 2024-10-07 14:00:00]. in 2024-11-13 08:00:04
|
||||
end dos_event part for time_range:2024-10-07 04:00:00(1728273600), 2024-10-07 14:00:00(1728309600), in 2024-11-13 08:00:04
|
||||
|
||||
start dos_event part for time_range:2024-10-06 18:00:00(1728237600), 2024-10-07 04:00:00(1728273600), in 2024-11-13 08:00:04
|
||||
同步数据段成功:[2024-10-06 18:00:00, 2024-10-07 04:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-06 18:00:00(1728237600), 2024-10-07 04:00:00(1728273600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-06 08:00:00(1728201600), 2024-10-06 18:00:00(1728237600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-06 08:00:00, 2024-10-06 18:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-06 08:00:00(1728201600), 2024-10-06 18:00:00(1728237600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-05 22:00:00(1728165600), 2024-10-06 08:00:00(1728201600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-05 22:00:00, 2024-10-06 08:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-05 22:00:00(1728165600), 2024-10-06 08:00:00(1728201600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-05 12:00:00(1728129600), 2024-10-05 22:00:00(1728165600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-05 12:00:00, 2024-10-05 22:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-05 12:00:00(1728129600), 2024-10-05 22:00:00(1728165600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-05 02:00:00(1728093600), 2024-10-05 12:00:00(1728129600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-05 02:00:00, 2024-10-05 12:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-05 02:00:00(1728093600), 2024-10-05 12:00:00(1728129600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-04 16:00:00(1728057600), 2024-10-05 02:00:00(1728093600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-04 16:00:00, 2024-10-05 02:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-04 16:00:00(1728057600), 2024-10-05 02:00:00(1728093600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-04 06:00:00(1728021600), 2024-10-04 16:00:00(1728057600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-04 06:00:00, 2024-10-04 16:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-04 06:00:00(1728021600), 2024-10-04 16:00:00(1728057600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-03 20:00:00(1727985600), 2024-10-04 06:00:00(1728021600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-03 20:00:00, 2024-10-04 06:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-03 20:00:00(1727985600), 2024-10-04 06:00:00(1728021600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-03 10:00:00(1727949600), 2024-10-03 20:00:00(1727985600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-03 10:00:00, 2024-10-03 20:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-03 10:00:00(1727949600), 2024-10-03 20:00:00(1727985600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-03 00:00:00(1727913600), 2024-10-03 10:00:00(1727949600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-03 00:00:00, 2024-10-03 10:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-03 00:00:00(1727913600), 2024-10-03 10:00:00(1727949600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-02 14:00:00(1727877600), 2024-10-03 00:00:00(1727913600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-02 14:00:00, 2024-10-03 00:00:00]. in 2024-11-13 08:00:05
|
||||
end dos_event part for time_range:2024-10-02 14:00:00(1727877600), 2024-10-03 00:00:00(1727913600), in 2024-11-13 08:00:05
|
||||
|
||||
start dos_event part for time_range:2024-10-02 04:00:00(1727841600), 2024-10-02 14:00:00(1727877600), in 2024-11-13 08:00:05
|
||||
同步数据段成功:[2024-10-02 04:00:00, 2024-10-02 14:00:00]. in 2024-11-13 08:00:06
|
||||
end dos_event part for time_range:2024-10-02 04:00:00(1727841600), 2024-10-02 14:00:00(1727877600), in 2024-11-13 08:00:06
|
||||
|
||||
start dos_event part for time_range:2024-10-01 18:00:00(1727805600), 2024-10-02 04:00:00(1727841600), in 2024-11-13 08:00:06
|
||||
同步数据段成功:[2024-10-01 18:00:00, 2024-10-02 04:00:00]. in 2024-11-13 08:00:06
|
||||
end dos_event part for time_range:2024-10-01 18:00:00(1727805600), 2024-10-02 04:00:00(1727841600), in 2024-11-13 08:00:06
|
||||
|
||||
start dos_event part for time_range:2024-10-01 08:00:00(1727769600), 2024-10-01 18:00:00(1727805600), in 2024-11-13 08:00:06
|
||||
同步数据段成功:[2024-10-01 08:00:00, 2024-10-01 18:00:00]. in 2024-11-13 08:00:06
|
||||
end dos_event part for time_range:2024-10-01 08:00:00(1727769600), 2024-10-01 18:00:00(1727805600), in 2024-11-13 08:00:06
|
||||
|
||||
start dos_event part for time_range:2024-10-01 00:00:00(1727740800), 2024-10-01 08:00:00(1727769600), in 2024-11-13 08:00:06
|
||||
同步数据段成功:[2024-10-01 00:00:00, 2024-10-01 08:00:00]. in 2024-11-13 08:00:06
|
||||
end dos_event part for time_range:2024-10-01 00:00:00(1727740800), 2024-10-01 08:00:00(1727769600), in 2024-11-13 08:00:06
|
||||
|
||||
**********end migrate dos_event table 2024-11-13 08:00:06**********
|
||||
migrate_table_end:2024-11-13 08:00:06, 成功迁移数据段:125, 失败迁移数据段:0
|
||||
成功迁移数据段:125, 失败迁移数据段:0
|
||||
#####################end 2024-11-13 08:00:06#######################
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据
|
||||
|
||||
# 迁移表 参数，可选值：assessment_event，dos_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 需要迁移的表名称
|
||||
|
||||
assessment_event_table_src="tsg_galaxy_tmp.assessment_event_local_old"
|
||||
assessment_event_table_dest="tsg_galaxy_tmp.assessment_event_local"
|
||||
|
||||
dos_event_table_src="tsg_galaxy_tmp.dos_event_local_old"
|
||||
dos_event_table_dest="tsg_galaxy_tmp.dos_event_local"
|
||||
|
||||
# ck客户端参数
|
||||
ip="127.0.0.1"
|
||||
ck_port="9001"
|
||||
ck_username="default"
|
||||
ck_password="galaxy2019"
|
||||
ck_max_execution_time=36000 # 执行insert 的超时时间秒
|
||||
max_threads=2 # max_insert_threads和max_threads
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "#####################start `date --utc +'%Y-%m-%d %H:%M:%S'`#######################"
|
||||
echo "migrate_table_start:`date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
echo "输入参数: table:$table"
|
||||
echo "输入参数: data_start_time:$data_start_time($timestamp_start)"
|
||||
echo "输入参数: data_end_time:$data_end_time($timestamp_end)"
|
||||
|
||||
insert_success_cnt=0
|
||||
insert_fail_cnt=0
|
||||
|
||||
|
||||
migrate_assessment_event_table_one_part(){
|
||||
input_time_start=$1
|
||||
input_time_end=$2
|
||||
insert_sql="insert into $assessment_event_table_dest
|
||||
SELECT
|
||||
*
|
||||
FROM $assessment_event_table_src
|
||||
where recv_time >= $input_time_start and recv_time < $input_time_end
|
||||
;"
|
||||
clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql"
|
||||
rst_code=$?
|
||||
start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"`
|
||||
end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"`
|
||||
if [ $rst_code -eq 0 ]; then
|
||||
insert_success_cnt=$(($insert_success_cnt+1))
|
||||
echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
else
|
||||
insert_fail_cnt=$(($insert_fail_cnt+1))
|
||||
echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
fi
|
||||
}
|
||||
|
||||
migrate_dos_event_table_one_part(){
|
||||
input_time_start=$1
|
||||
input_time_end=$2
|
||||
insert_sql="insert into $dos_event_table_dest
|
||||
SELECT
|
||||
*
|
||||
FROM $dos_event_table_src
|
||||
where recv_time >= $input_time_start and recv_time < $input_time_end
|
||||
;"
|
||||
clickhouse-client -h $ip --port $ck_port -m -n -u $ck_username --password $ck_password --max_insert_threads=$max_threads --max_threads=$max_threads --max_execution_time=$ck_max_execution_time --query="$insert_sql"
|
||||
rst_code=$?
|
||||
start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"`
|
||||
end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"`
|
||||
if [ $rst_code -eq 0 ]; then
|
||||
insert_success_cnt=$(($insert_success_cnt+1))
|
||||
echo "同步数据段成功:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
else
|
||||
insert_fail_cnt=$(($insert_fail_cnt+1))
|
||||
echo "同步数据段失败:[${start_time}, ${end_time}]. in `date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
fi
|
||||
}
|
||||
|
||||
slice_time_interval=$((60*slice_interval_minute))
|
||||
data_split_part=$(((timestamp_end-timestamp_start)/slice_time_interval))
|
||||
|
||||
# 输出划分的时间段
|
||||
show_time_range(){
|
||||
echo "split time ranges:"
|
||||
for ((i = 0 ; i <= $data_split_part ; i++))
|
||||
do
|
||||
input_time_end=$((timestamp_end-i*slice_time_interval))
|
||||
input_time_start=$((input_time_end-slice_time_interval))
|
||||
|
||||
if [ $i -eq $data_split_part ]; then
|
||||
# 正好均分
|
||||
if [ $input_time_end -le $timestamp_start ]; then
|
||||
continue
|
||||
fi
|
||||
input_time_start=$timestamp_start
|
||||
fi
|
||||
|
||||
|
||||
start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"`
|
||||
end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"`
|
||||
echo "time_range:$start_time, $end_time"
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
|
||||
# 迁移assessment_event
|
||||
migrate_assessment_event_table(){
|
||||
echo "**********start migrate $table table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********"
|
||||
|
||||
# 遍历每个时间段数据
|
||||
for ((i = 0 ; i <= $data_split_part ; i++))
|
||||
do
|
||||
input_time_end=$((timestamp_end-i*slice_time_interval))
|
||||
input_time_start=$((input_time_end-slice_time_interval))
|
||||
|
||||
if [ $i -eq $data_split_part ]; then
|
||||
# 正好均分
|
||||
if [ $input_time_end -le $timestamp_start ]; then
|
||||
continue
|
||||
fi
|
||||
input_time_start=$timestamp_start
|
||||
fi
|
||||
|
||||
|
||||
start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"`
|
||||
end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"`
|
||||
|
||||
echo "start $table part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
# 迁移此时间段数据
|
||||
migrate_assessment_event_table_one_part $input_time_start $input_time_end
|
||||
echo "end $table part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "**********end migrate $table table `date --utc +'%Y-%m-%d %H:%M:%S'`**********"
|
||||
}
|
||||
|
||||
# 迁移dos_event
|
||||
migrate_dos_event_table(){
|
||||
echo "**********start migrate $table table in `date --utc +'%Y-%m-%d %H:%M:%S'`**********"
|
||||
|
||||
# 遍历每个时间段数据
|
||||
for ((i = 0 ; i <= $data_split_part ; i++))
|
||||
do
|
||||
input_time_end=$((timestamp_end-i*slice_time_interval))
|
||||
input_time_start=$((input_time_end-slice_time_interval))
|
||||
|
||||
if [ $i -eq $data_split_part ]; then
|
||||
# 正好均分
|
||||
if [ $input_time_end -le $timestamp_start ]; then
|
||||
continue
|
||||
fi
|
||||
input_time_start=$timestamp_start
|
||||
fi
|
||||
|
||||
|
||||
start_time=`date --utc -d @$input_time_start "+%Y-%m-%d %H:%M:%S"`
|
||||
end_time=`date --utc -d @$input_time_end "+%Y-%m-%d %H:%M:%S"`
|
||||
|
||||
echo "start $table part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
# 迁移此时间段数据
|
||||
migrate_dos_event_table_one_part $input_time_start $input_time_end
|
||||
echo "end $table part for time_range:${start_time}(${input_time_start}), ${end_time}(${input_time_end}), in `date --utc +'%Y-%m-%d %H:%M:%S'`"
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "**********end migrate $table table `date --utc +'%Y-%m-%d %H:%M:%S'`**********"
|
||||
}
|
||||
|
||||
# 输出划分的时间段
|
||||
show_time_range
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"assessment_event")
|
||||
# 迁移assessment_event
|
||||
migrate_assessment_event_table
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
migrate_dos_event_table
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:assessment_event，dos_event"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "migrate_table_end:`date --utc +'%Y-%m-%d %H:%M:%S'`, 成功迁移数据段:${insert_success_cnt}, 失败迁移数据段:${insert_fail_cnt}"
|
||||
|
||||
echo "成功迁移数据段:${insert_success_cnt}, 失败迁移数据段:${insert_fail_cnt}"
|
||||
|
||||
echo "#####################end `date --utc +'%Y-%m-%d %H:%M:%S'`#######################"
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 本脚本逐时间段按最新往前顺序迁移clickhouse数据，TSG24.01日志重组后数据迁移
|
||||
|
||||
# 迁移表 参数，可选值：assessment_event，dos_event
|
||||
table=$1
|
||||
# 数据开始时间(UTC) 参数, 例如:"2023-10-26 00:00:00"
|
||||
data_start_time=$2
|
||||
# 数据结束时间(UTC) 参数, 例如:"2023-10-28 00:00:00"
|
||||
data_end_time=$3
|
||||
# 每批迁移数据时间段长度(分钟) 参数, 例如:240
|
||||
slice_interval_minute=$4
|
||||
|
||||
timestamp_start=`date --utc --date="$data_start_time" +%s`
|
||||
timestamp_end=`date --utc --date="$data_end_time" +%s`
|
||||
|
||||
# 校验迁移表参数
|
||||
case $table in
|
||||
"assessment_event")
|
||||
# 迁移assessment_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
"dos_event")
|
||||
# 迁移dos_event
|
||||
echo "迁移表${table}, start:$data_start_time, end:$data_end_time, interval_minute:$slice_interval_minute"
|
||||
;;
|
||||
*)
|
||||
echo "所迁移表${table}不在范围:assessment_event，dos_event"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# 校验时间参数
|
||||
if [ -z "$timestamp_start" ]; then
|
||||
echo "data_start_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$timestamp_end" ]; then
|
||||
echo "data_end_time fmt err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $timestamp_start -ge $timestamp_end ]; then
|
||||
echo "date range err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "$slice_interval_minute" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "slice_interval_minute参数必须是正确的分钟数"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 后台执行，输出日志
|
||||
nohup ./migrate_local_table_to_2411.sh "$table" "$data_start_time" "$data_end_time" $slice_interval_minute > "log_$table.txt" 2>&1 &
|
||||
echo "已启动迁移${table}表任务，时间范围[$data_start_time, $data_end_time], 每批迁移段分钟:$slice_interval_minute, 日志输出到:log_$table.txt。请查看日志文件确认每段数据迁移情况"
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
create database if not exists tsg_galaxy_tmp on cluster ck_cluster;
|
||||
|
||||
|
||||
-- 源表rename到历史表
|
||||
|
||||
|
||||
ALTER table tsg_galaxy_tmp.assessment_event_local_old on cluster ck_cluster add column IF NOT EXISTS log_uuid String after log_id;
|
||||
ALTER table tsg_galaxy_tmp.dos_event_local_old on cluster ck_cluster add column IF NOT EXISTS log_uuid String after log_id;
|
||||
|
||||
|
||||
-- 创建源分布式表old
|
||||
create table IF NOT EXISTS tsg_galaxy_tmp.assessment_event_old ON CLUSTER ck_cluster (
|
||||
log_id UInt64,
|
||||
recv_time Int64
|
||||
) ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,assessment_event_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_old ON CLUSTER ck_cluster(
|
||||
log_id UInt64,
|
||||
recv_time Int64,
|
||||
) ENGINE = Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local_old,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event_local on cluster ck_cluster (
|
||||
vsys_id Int32,
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
rule_id Int64,
|
||||
rule_uuid String,
|
||||
start_time Int64,
|
||||
end_time Int64,
|
||||
attack_type String,
|
||||
severity String,
|
||||
conditions String,
|
||||
source_ip String,
|
||||
source_country String,
|
||||
destination_ip String,
|
||||
destination_country String,
|
||||
source_ip_list String,
|
||||
source_country_list String,
|
||||
sessions Int64,
|
||||
session_rate Float64,
|
||||
packets Int64,
|
||||
packet_rate Float64,
|
||||
bytes Int64,
|
||||
bit_rate Float64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,destination_ip,recv_time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.dos_event on cluster ck_cluster (
|
||||
vsys_id Int32,
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
rule_id Int64,
|
||||
rule_uuid String,
|
||||
start_time Int64,
|
||||
end_time Int64,
|
||||
attack_type String,
|
||||
severity String,
|
||||
conditions String,
|
||||
source_ip String,
|
||||
source_country String,
|
||||
destination_ip String,
|
||||
destination_country String,
|
||||
source_ip_list String,
|
||||
source_country_list String,
|
||||
sessions Int64,
|
||||
session_rate Float64,
|
||||
packets Int64,
|
||||
packet_rate Float64,
|
||||
bytes Int64,
|
||||
bit_rate Float64
|
||||
)
|
||||
ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,dos_event_local,rand());
|
||||
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.assessment_event_local on cluster ck_cluster (
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
recv_time Int64,
|
||||
vsys_id Int64,
|
||||
assessment_date Int64,
|
||||
lot_number String,
|
||||
file_name String,
|
||||
assessment_file String,
|
||||
assessment_type String,
|
||||
features String,
|
||||
size Int64,
|
||||
file_checksum_sha String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,recv_time);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.assessment_event on cluster ck_cluster (
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
recv_time Int64,
|
||||
vsys_id Int64,
|
||||
assessment_date Int64,
|
||||
lot_number String,
|
||||
file_name String,
|
||||
assessment_file String,
|
||||
assessment_type String,
|
||||
features String,
|
||||
size Int64,
|
||||
file_checksum_sha String
|
||||
)
|
||||
ENGINE =Distributed(ck_cluster,tsg_galaxy_tmp,assessment_event_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record_local on cluster ck_cluster (
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
decoded_as String,
|
||||
session_id UInt64,
|
||||
session_uuid String,
|
||||
ingestion_time Int64,
|
||||
processing_time Int64,
|
||||
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
|
||||
address_type Int32,
|
||||
vsys_id Int32,
|
||||
client_ip String,
|
||||
client_port Int32,
|
||||
server_ip String,
|
||||
server_port Int32,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
dns_message_id Nullable(Int32),
|
||||
dns_qr Nullable(Int32),
|
||||
dns_opcode Nullable(Int32),
|
||||
dns_aa Nullable(Int32),
|
||||
dns_tc Nullable(Int32),
|
||||
dns_rd Nullable(Int32),
|
||||
dns_ra Nullable(Int32),
|
||||
dns_rcode Nullable(Int32),
|
||||
dns_qdcount Nullable(Int32),
|
||||
dns_ancount Nullable(Int32),
|
||||
dns_nscount Nullable(Int32),
|
||||
dns_arcount Nullable(Int32),
|
||||
dns_qname String,
|
||||
dns_qtype Nullable(Int32),
|
||||
dns_qclass Nullable(Int32),
|
||||
dns_cname String,
|
||||
dns_sub Nullable(Int32),
|
||||
dns_rr String,
|
||||
dns_response_latency_ms Nullable(Int32),
|
||||
http_url String,
|
||||
http_host String,
|
||||
http_request_line String,
|
||||
http_response_line String,
|
||||
http_request_body String,
|
||||
http_response_body String,
|
||||
http_proxy_flag Nullable(Int32),
|
||||
http_sequence Nullable(Int32),
|
||||
http_cookie String,
|
||||
http_referer String,
|
||||
http_user_agent String,
|
||||
http_request_content_length Nullable(Int64),
|
||||
http_request_content_type String,
|
||||
http_response_content_length Nullable(Int64),
|
||||
http_response_content_type String,
|
||||
http_set_cookie String,
|
||||
http_version String,
|
||||
http_status_code Nullable(Int32),
|
||||
http_response_latency_ms Nullable(Int32),
|
||||
http_session_duration_ms Nullable(Int32),
|
||||
http_action_file_size Nullable(Int64),
|
||||
mail_protocol_type String,
|
||||
mail_account String,
|
||||
mail_from_cmd String,
|
||||
mail_to_cmd String,
|
||||
mail_from String,
|
||||
mail_password String,
|
||||
mail_to String,
|
||||
mail_cc String,
|
||||
mail_bcc String,
|
||||
mail_subject String,
|
||||
mail_subject_charset String,
|
||||
mail_attachment_name String,
|
||||
mail_attachment_name_charset String,
|
||||
mail_starttls_flag Nullable(Int32),
|
||||
mail_eml_file String,
|
||||
sip_call_id String,
|
||||
sip_originator_description String,
|
||||
sip_responder_description String,
|
||||
sip_user_agent String,
|
||||
sip_server String,
|
||||
sip_originator_sdp_connect_ip String,
|
||||
sip_originator_sdp_media_port Nullable(Int32),
|
||||
sip_originator_sdp_media_type String,
|
||||
sip_originator_sdp_content String,
|
||||
sip_responder_sdp_connect_ip String,
|
||||
sip_responder_sdp_media_port Nullable(Int32),
|
||||
sip_responder_sdp_media_type String,
|
||||
sip_responder_sdp_content String,
|
||||
sip_duration_s Nullable(Int32),
|
||||
sip_bye String,
|
||||
sip_bye_reason String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,session_uuid,recv_time);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_tmp.transaction_record on cluster ck_cluster (
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
decoded_as String,
|
||||
session_id UInt64,
|
||||
session_uuid String,
|
||||
ingestion_time Int64,
|
||||
processing_time Int64,
|
||||
insert_time Int64 ,
|
||||
address_type Int32,
|
||||
vsys_id Int32,
|
||||
client_ip String,
|
||||
client_port Int32,
|
||||
server_ip String,
|
||||
server_port Int32,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
dns_message_id Nullable(Int32),
|
||||
dns_qr Nullable(Int32),
|
||||
dns_opcode Nullable(Int32),
|
||||
dns_aa Nullable(Int32),
|
||||
dns_tc Nullable(Int32),
|
||||
dns_rd Nullable(Int32),
|
||||
dns_ra Nullable(Int32),
|
||||
dns_rcode Nullable(Int32),
|
||||
dns_qdcount Nullable(Int32),
|
||||
dns_ancount Nullable(Int32),
|
||||
dns_nscount Nullable(Int32),
|
||||
dns_arcount Nullable(Int32),
|
||||
dns_qname String,
|
||||
dns_qtype Nullable(Int32),
|
||||
dns_qclass Nullable(Int32),
|
||||
dns_cname String,
|
||||
dns_sub Nullable(Int32),
|
||||
dns_rr String,
|
||||
dns_response_latency_ms Nullable(Int32),
|
||||
http_url String,
|
||||
http_host String,
|
||||
http_request_line String,
|
||||
http_response_line String,
|
||||
http_request_body String,
|
||||
http_response_body String,
|
||||
http_proxy_flag Nullable(Int32),
|
||||
http_sequence Nullable(Int32),
|
||||
http_cookie String,
|
||||
http_referer String,
|
||||
http_user_agent String,
|
||||
http_request_content_length Nullable(Int64),
|
||||
http_request_content_type String,
|
||||
http_response_content_length Nullable(Int64),
|
||||
http_response_content_type String,
|
||||
http_set_cookie String,
|
||||
http_version String,
|
||||
http_status_code Nullable(Int32),
|
||||
http_response_latency_ms Nullable(Int32),
|
||||
http_session_duration_ms Nullable(Int32),
|
||||
http_action_file_size Nullable(Int64),
|
||||
mail_protocol_type String,
|
||||
mail_account String,
|
||||
mail_from_cmd String,
|
||||
mail_to_cmd String,
|
||||
mail_from String,
|
||||
mail_password String,
|
||||
mail_to String,
|
||||
mail_cc String,
|
||||
mail_bcc String,
|
||||
mail_subject String,
|
||||
mail_subject_charset String,
|
||||
mail_attachment_name String,
|
||||
mail_attachment_name_charset String,
|
||||
mail_starttls_flag Nullable(Int32),
|
||||
mail_eml_file String,
|
||||
sip_call_id String,
|
||||
sip_originator_description String,
|
||||
sip_responder_description String,
|
||||
sip_user_agent String,
|
||||
sip_server String,
|
||||
sip_originator_sdp_connect_ip String,
|
||||
sip_originator_sdp_media_port Nullable(Int32),
|
||||
sip_originator_sdp_media_type String,
|
||||
sip_originator_sdp_content String,
|
||||
sip_responder_sdp_connect_ip String,
|
||||
sip_responder_sdp_media_port Nullable(Int32),
|
||||
sip_responder_sdp_media_type String,
|
||||
sip_responder_sdp_content String,
|
||||
sip_duration_s Nullable(Int32),
|
||||
sip_bye String,
|
||||
sip_bye_reason String
|
||||
)
|
||||
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand());
|
||||
@@ -0,0 +1,296 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
drop TABLE IF EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster ;
|
||||
drop TABLE IF EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster ;
|
||||
drop TABLE IF EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster ;
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster (
|
||||
vsys_id Int32,
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
rule_id Int64,
|
||||
rule_uuid String,
|
||||
start_time Int64,
|
||||
end_time Int64,
|
||||
attack_type String,
|
||||
severity String,
|
||||
conditions String,
|
||||
source_ip String,
|
||||
source_country String,
|
||||
destination_ip String,
|
||||
destination_country String,
|
||||
source_ip_list String,
|
||||
source_country_list String,
|
||||
sessions Int64,
|
||||
session_rate Float64,
|
||||
packets Int64,
|
||||
packet_rate Float64,
|
||||
bytes Int64,
|
||||
bit_rate Float64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,destination_ip,recv_time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster (
|
||||
vsys_id Int32,
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
rule_id Int64,
|
||||
rule_uuid String,
|
||||
start_time Int64,
|
||||
end_time Int64,
|
||||
attack_type String,
|
||||
severity String,
|
||||
conditions String,
|
||||
source_ip String,
|
||||
source_country String,
|
||||
destination_ip String,
|
||||
destination_country String,
|
||||
source_ip_list String,
|
||||
source_country_list String,
|
||||
sessions Int64,
|
||||
session_rate Float64,
|
||||
packets Int64,
|
||||
packet_rate Float64,
|
||||
bytes Int64,
|
||||
bit_rate Float64
|
||||
)
|
||||
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand());
|
||||
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster (
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
recv_time Int64,
|
||||
vsys_id Int64,
|
||||
assessment_date Int64,
|
||||
lot_number String,
|
||||
file_name String,
|
||||
assessment_file String,
|
||||
assessment_type String,
|
||||
features String,
|
||||
size Int64,
|
||||
file_checksum_sha String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,recv_time);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster (
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
recv_time Int64,
|
||||
vsys_id Int64,
|
||||
assessment_date Int64,
|
||||
lot_number String,
|
||||
file_name String,
|
||||
assessment_file String,
|
||||
assessment_type String,
|
||||
features String,
|
||||
size Int64,
|
||||
file_checksum_sha String
|
||||
)
|
||||
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster (
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
decoded_as String,
|
||||
session_id UInt64,
|
||||
session_uuid String,
|
||||
ingestion_time Int64,
|
||||
processing_time Int64,
|
||||
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
|
||||
address_type Int32,
|
||||
vsys_id Int32,
|
||||
client_ip String,
|
||||
client_port Int32,
|
||||
server_ip String,
|
||||
server_port Int32,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
dns_message_id Nullable(Int32),
|
||||
dns_qr Nullable(Int32),
|
||||
dns_opcode Nullable(Int32),
|
||||
dns_aa Nullable(Int32),
|
||||
dns_tc Nullable(Int32),
|
||||
dns_rd Nullable(Int32),
|
||||
dns_ra Nullable(Int32),
|
||||
dns_rcode Nullable(Int32),
|
||||
dns_qdcount Nullable(Int32),
|
||||
dns_ancount Nullable(Int32),
|
||||
dns_nscount Nullable(Int32),
|
||||
dns_arcount Nullable(Int32),
|
||||
dns_qname String,
|
||||
dns_qtype Nullable(Int32),
|
||||
dns_qclass Nullable(Int32),
|
||||
dns_cname String,
|
||||
dns_sub Nullable(Int32),
|
||||
dns_rr String,
|
||||
dns_response_latency_ms Nullable(Int32),
|
||||
http_url String,
|
||||
http_host String,
|
||||
http_request_line String,
|
||||
http_response_line String,
|
||||
http_request_body String,
|
||||
http_response_body String,
|
||||
http_proxy_flag Nullable(Int32),
|
||||
http_sequence Nullable(Int32),
|
||||
http_cookie String,
|
||||
http_referer String,
|
||||
http_user_agent String,
|
||||
http_request_content_length Nullable(Int64),
|
||||
http_request_content_type String,
|
||||
http_response_content_length Nullable(Int64),
|
||||
http_response_content_type String,
|
||||
http_set_cookie String,
|
||||
http_version String,
|
||||
http_status_code Nullable(Int32),
|
||||
http_response_latency_ms Nullable(Int32),
|
||||
http_session_duration_ms Nullable(Int32),
|
||||
http_action_file_size Nullable(Int64),
|
||||
mail_protocol_type String,
|
||||
mail_account String,
|
||||
mail_from_cmd String,
|
||||
mail_to_cmd String,
|
||||
mail_from String,
|
||||
mail_password String,
|
||||
mail_to String,
|
||||
mail_cc String,
|
||||
mail_bcc String,
|
||||
mail_subject String,
|
||||
mail_subject_charset String,
|
||||
mail_attachment_name String,
|
||||
mail_attachment_name_charset String,
|
||||
mail_starttls_flag Nullable(Int32),
|
||||
mail_eml_file String,
|
||||
sip_call_id String,
|
||||
sip_originator_description String,
|
||||
sip_responder_description String,
|
||||
sip_user_agent String,
|
||||
sip_server String,
|
||||
sip_originator_sdp_connect_ip String,
|
||||
sip_originator_sdp_media_port Nullable(Int32),
|
||||
sip_originator_sdp_media_type String,
|
||||
sip_originator_sdp_content String,
|
||||
sip_responder_sdp_connect_ip String,
|
||||
sip_responder_sdp_media_port Nullable(Int32),
|
||||
sip_responder_sdp_media_type String,
|
||||
sip_responder_sdp_content String,
|
||||
sip_duration_s Nullable(Int32),
|
||||
sip_bye String,
|
||||
sip_bye_reason String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,session_uuid,recv_time);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster (
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
decoded_as String,
|
||||
session_id UInt64,
|
||||
session_uuid String,
|
||||
ingestion_time Int64,
|
||||
processing_time Int64,
|
||||
insert_time Int64 ,
|
||||
address_type Int32,
|
||||
vsys_id Int32,
|
||||
client_ip String,
|
||||
client_port Int32,
|
||||
server_ip String,
|
||||
server_port Int32,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
dns_message_id Nullable(Int32),
|
||||
dns_qr Nullable(Int32),
|
||||
dns_opcode Nullable(Int32),
|
||||
dns_aa Nullable(Int32),
|
||||
dns_tc Nullable(Int32),
|
||||
dns_rd Nullable(Int32),
|
||||
dns_ra Nullable(Int32),
|
||||
dns_rcode Nullable(Int32),
|
||||
dns_qdcount Nullable(Int32),
|
||||
dns_ancount Nullable(Int32),
|
||||
dns_nscount Nullable(Int32),
|
||||
dns_arcount Nullable(Int32),
|
||||
dns_qname String,
|
||||
dns_qtype Nullable(Int32),
|
||||
dns_qclass Nullable(Int32),
|
||||
dns_cname String,
|
||||
dns_sub Nullable(Int32),
|
||||
dns_rr String,
|
||||
dns_response_latency_ms Nullable(Int32),
|
||||
http_url String,
|
||||
http_host String,
|
||||
http_request_line String,
|
||||
http_response_line String,
|
||||
http_request_body String,
|
||||
http_response_body String,
|
||||
http_proxy_flag Nullable(Int32),
|
||||
http_sequence Nullable(Int32),
|
||||
http_cookie String,
|
||||
http_referer String,
|
||||
http_user_agent String,
|
||||
http_request_content_length Nullable(Int64),
|
||||
http_request_content_type String,
|
||||
http_response_content_length Nullable(Int64),
|
||||
http_response_content_type String,
|
||||
http_set_cookie String,
|
||||
http_version String,
|
||||
http_status_code Nullable(Int32),
|
||||
http_response_latency_ms Nullable(Int32),
|
||||
http_session_duration_ms Nullable(Int32),
|
||||
http_action_file_size Nullable(Int64),
|
||||
mail_protocol_type String,
|
||||
mail_account String,
|
||||
mail_from_cmd String,
|
||||
mail_to_cmd String,
|
||||
mail_from String,
|
||||
mail_password String,
|
||||
mail_to String,
|
||||
mail_cc String,
|
||||
mail_bcc String,
|
||||
mail_subject String,
|
||||
mail_subject_charset String,
|
||||
mail_attachment_name String,
|
||||
mail_attachment_name_charset String,
|
||||
mail_starttls_flag Nullable(Int32),
|
||||
mail_eml_file String,
|
||||
sip_call_id String,
|
||||
sip_originator_description String,
|
||||
sip_responder_description String,
|
||||
sip_user_agent String,
|
||||
sip_server String,
|
||||
sip_originator_sdp_connect_ip String,
|
||||
sip_originator_sdp_media_port Nullable(Int32),
|
||||
sip_originator_sdp_media_type String,
|
||||
sip_originator_sdp_content String,
|
||||
sip_responder_sdp_connect_ip String,
|
||||
sip_responder_sdp_media_port Nullable(Int32),
|
||||
sip_responder_sdp_media_type String,
|
||||
sip_responder_sdp_content String,
|
||||
sip_duration_s Nullable(Int32),
|
||||
sip_bye String,
|
||||
sip_bye_reason String
|
||||
)
|
||||
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand());
|
||||
|
||||
|
||||
|
||||
alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1;
|
||||
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster;
|
||||
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster (
|
||||
vsys_id Int32,
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
rule_id Int64,
|
||||
rule_uuid String,
|
||||
start_time Int64,
|
||||
@@ -26,12 +29,13 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster (
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,destination_ip,recv_time,log_id);
|
||||
ORDER BY (vsys_id,destination_ip,recv_time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster (
|
||||
vsys_id Int32,
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
rule_id Int64,
|
||||
rule_uuid String,
|
||||
start_time Int64,
|
||||
@@ -58,6 +62,7 @@ ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand());
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster (
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
recv_time Int64,
|
||||
vsys_id Int64,
|
||||
assessment_date Int64,
|
||||
@@ -71,11 +76,12 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cl
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,recv_time,log_id);
|
||||
ORDER BY (vsys_id,recv_time);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster (
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
recv_time Int64,
|
||||
vsys_id Int64,
|
||||
assessment_date Int64,
|
||||
@@ -89,9 +95,6 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster
|
||||
)
|
||||
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand());
|
||||
|
||||
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster (
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
@@ -1681,194 +1684,196 @@ ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand());
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster (
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
decoded_as String,
|
||||
session_id UInt64,
|
||||
ingestion_time Int64,
|
||||
processing_time Int64,
|
||||
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
|
||||
address_type Int32,
|
||||
vsys_id Int32,
|
||||
client_ip String,
|
||||
client_port Int32,
|
||||
server_ip String,
|
||||
server_port Int32,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
dns_message_id Nullable(Int32),
|
||||
dns_qr Nullable(Int32),
|
||||
dns_opcode Nullable(Int32),
|
||||
dns_aa Nullable(Int32),
|
||||
dns_tc Nullable(Int32),
|
||||
dns_rd Nullable(Int32),
|
||||
dns_ra Nullable(Int32),
|
||||
dns_rcode Nullable(Int32),
|
||||
dns_qdcount Nullable(Int32),
|
||||
dns_ancount Nullable(Int32),
|
||||
dns_nscount Nullable(Int32),
|
||||
dns_arcount Nullable(Int32),
|
||||
dns_qname String,
|
||||
dns_qtype Nullable(Int32),
|
||||
dns_qclass Nullable(Int32),
|
||||
dns_cname String,
|
||||
dns_sub Nullable(Int32),
|
||||
dns_rr String,
|
||||
dns_response_latency_ms Nullable(Int32),
|
||||
http_url String,
|
||||
http_host String,
|
||||
http_request_line String,
|
||||
http_response_line String,
|
||||
http_request_body String,
|
||||
http_response_body String,
|
||||
http_proxy_flag Nullable(Int32),
|
||||
http_sequence Nullable(Int32),
|
||||
http_cookie String,
|
||||
http_referer String,
|
||||
http_user_agent String,
|
||||
http_request_content_length Nullable(Int64),
|
||||
http_request_content_type String,
|
||||
http_response_content_length Nullable(Int64),
|
||||
http_response_content_type String,
|
||||
http_set_cookie String,
|
||||
http_version String,
|
||||
http_status_code Nullable(Int32),
|
||||
http_response_latency_ms Nullable(Int32),
|
||||
http_session_duration_ms Nullable(Int32),
|
||||
http_action_file_size Nullable(Int64),
|
||||
mail_protocol_type String,
|
||||
mail_account String,
|
||||
mail_from_cmd String,
|
||||
mail_to_cmd String,
|
||||
mail_from String,
|
||||
mail_password String,
|
||||
mail_to String,
|
||||
mail_cc String,
|
||||
mail_bcc String,
|
||||
mail_subject String,
|
||||
mail_subject_charset String,
|
||||
mail_attachment_name String,
|
||||
mail_attachment_name_charset String,
|
||||
mail_starttls_flag Nullable(Int32),
|
||||
mail_eml_file String,
|
||||
sip_call_id String,
|
||||
sip_originator_description String,
|
||||
sip_responder_description String,
|
||||
sip_user_agent String,
|
||||
sip_server String,
|
||||
sip_originator_sdp_connect_ip String,
|
||||
sip_originator_sdp_media_port Nullable(Int32),
|
||||
sip_originator_sdp_media_type String,
|
||||
sip_originator_sdp_content String,
|
||||
sip_responder_sdp_connect_ip String,
|
||||
sip_responder_sdp_media_port Nullable(Int32),
|
||||
sip_responder_sdp_media_type String,
|
||||
sip_responder_sdp_content String,
|
||||
sip_duration_s Nullable(Int32),
|
||||
sip_bye String,
|
||||
sip_bye_reason String
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
decoded_as String,
|
||||
session_id UInt64,
|
||||
session_uuid String,
|
||||
ingestion_time Int64,
|
||||
processing_time Int64,
|
||||
insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
|
||||
address_type Int32,
|
||||
vsys_id Int32,
|
||||
client_ip String,
|
||||
client_port Int32,
|
||||
server_ip String,
|
||||
server_port Int32,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
dns_message_id Nullable(Int32),
|
||||
dns_qr Nullable(Int32),
|
||||
dns_opcode Nullable(Int32),
|
||||
dns_aa Nullable(Int32),
|
||||
dns_tc Nullable(Int32),
|
||||
dns_rd Nullable(Int32),
|
||||
dns_ra Nullable(Int32),
|
||||
dns_rcode Nullable(Int32),
|
||||
dns_qdcount Nullable(Int32),
|
||||
dns_ancount Nullable(Int32),
|
||||
dns_nscount Nullable(Int32),
|
||||
dns_arcount Nullable(Int32),
|
||||
dns_qname String,
|
||||
dns_qtype Nullable(Int32),
|
||||
dns_qclass Nullable(Int32),
|
||||
dns_cname String,
|
||||
dns_sub Nullable(Int32),
|
||||
dns_rr String,
|
||||
dns_response_latency_ms Nullable(Int32),
|
||||
http_url String,
|
||||
http_host String,
|
||||
http_request_line String,
|
||||
http_response_line String,
|
||||
http_request_body String,
|
||||
http_response_body String,
|
||||
http_proxy_flag Nullable(Int32),
|
||||
http_sequence Nullable(Int32),
|
||||
http_cookie String,
|
||||
http_referer String,
|
||||
http_user_agent String,
|
||||
http_request_content_length Nullable(Int64),
|
||||
http_request_content_type String,
|
||||
http_response_content_length Nullable(Int64),
|
||||
http_response_content_type String,
|
||||
http_set_cookie String,
|
||||
http_version String,
|
||||
http_status_code Nullable(Int32),
|
||||
http_response_latency_ms Nullable(Int32),
|
||||
http_session_duration_ms Nullable(Int32),
|
||||
http_action_file_size Nullable(Int64),
|
||||
mail_protocol_type String,
|
||||
mail_account String,
|
||||
mail_from_cmd String,
|
||||
mail_to_cmd String,
|
||||
mail_from String,
|
||||
mail_password String,
|
||||
mail_to String,
|
||||
mail_cc String,
|
||||
mail_bcc String,
|
||||
mail_subject String,
|
||||
mail_subject_charset String,
|
||||
mail_attachment_name String,
|
||||
mail_attachment_name_charset String,
|
||||
mail_starttls_flag Nullable(Int32),
|
||||
mail_eml_file String,
|
||||
sip_call_id String,
|
||||
sip_originator_description String,
|
||||
sip_responder_description String,
|
||||
sip_user_agent String,
|
||||
sip_server String,
|
||||
sip_originator_sdp_connect_ip String,
|
||||
sip_originator_sdp_media_port Nullable(Int32),
|
||||
sip_originator_sdp_media_type String,
|
||||
sip_originator_sdp_content String,
|
||||
sip_responder_sdp_connect_ip String,
|
||||
sip_responder_sdp_media_port Nullable(Int32),
|
||||
sip_responder_sdp_media_type String,
|
||||
sip_responder_sdp_content String,
|
||||
sip_duration_s Nullable(Int32),
|
||||
sip_bye String,
|
||||
sip_bye_reason String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,session_id,recv_time);
|
||||
ORDER BY (vsys_id,session_uuid,recv_time);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster (
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
decoded_as String,
|
||||
session_id UInt64,
|
||||
ingestion_time Int64,
|
||||
processing_time Int64,
|
||||
insert_time Int64 ,
|
||||
address_type Int32,
|
||||
vsys_id Int32,
|
||||
client_ip String,
|
||||
client_port Int32,
|
||||
server_ip String,
|
||||
server_port Int32,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
dns_message_id Nullable(Int32),
|
||||
dns_qr Nullable(Int32),
|
||||
dns_opcode Nullable(Int32),
|
||||
dns_aa Nullable(Int32),
|
||||
dns_tc Nullable(Int32),
|
||||
dns_rd Nullable(Int32),
|
||||
dns_ra Nullable(Int32),
|
||||
dns_rcode Nullable(Int32),
|
||||
dns_qdcount Nullable(Int32),
|
||||
dns_ancount Nullable(Int32),
|
||||
dns_nscount Nullable(Int32),
|
||||
dns_arcount Nullable(Int32),
|
||||
dns_qname String,
|
||||
dns_qtype Nullable(Int32),
|
||||
dns_qclass Nullable(Int32),
|
||||
dns_cname String,
|
||||
dns_sub Nullable(Int32),
|
||||
dns_rr String,
|
||||
dns_response_latency_ms Nullable(Int32),
|
||||
http_url String,
|
||||
http_host String,
|
||||
http_request_line String,
|
||||
http_response_line String,
|
||||
http_request_body String,
|
||||
http_response_body String,
|
||||
http_proxy_flag Nullable(Int32),
|
||||
http_sequence Nullable(Int32),
|
||||
http_cookie String,
|
||||
http_referer String,
|
||||
http_user_agent String,
|
||||
http_request_content_length Nullable(Int64),
|
||||
http_request_content_type String,
|
||||
http_response_content_length Nullable(Int64),
|
||||
http_response_content_type String,
|
||||
http_set_cookie String,
|
||||
http_version String,
|
||||
http_status_code Nullable(Int32),
|
||||
http_response_latency_ms Nullable(Int32),
|
||||
http_session_duration_ms Nullable(Int32),
|
||||
http_action_file_size Nullable(Int64),
|
||||
mail_protocol_type String,
|
||||
mail_account String,
|
||||
mail_from_cmd String,
|
||||
mail_to_cmd String,
|
||||
mail_from String,
|
||||
mail_password String,
|
||||
mail_to String,
|
||||
mail_cc String,
|
||||
mail_bcc String,
|
||||
mail_subject String,
|
||||
mail_subject_charset String,
|
||||
mail_attachment_name String,
|
||||
mail_attachment_name_charset String,
|
||||
mail_starttls_flag Nullable(Int32),
|
||||
mail_eml_file String,
|
||||
sip_call_id String,
|
||||
sip_originator_description String,
|
||||
sip_responder_description String,
|
||||
sip_user_agent String,
|
||||
sip_server String,
|
||||
sip_originator_sdp_connect_ip String,
|
||||
sip_originator_sdp_media_port Nullable(Int32),
|
||||
sip_originator_sdp_media_type String,
|
||||
sip_originator_sdp_content String,
|
||||
sip_responder_sdp_connect_ip String,
|
||||
sip_responder_sdp_media_port Nullable(Int32),
|
||||
sip_responder_sdp_media_type String,
|
||||
sip_responder_sdp_content String,
|
||||
sip_duration_s Nullable(Int32),
|
||||
sip_bye String,
|
||||
sip_bye_reason String
|
||||
recv_time Int64,
|
||||
log_id UInt64,
|
||||
log_uuid String,
|
||||
decoded_as String,
|
||||
session_id UInt64,
|
||||
session_uuid String,
|
||||
ingestion_time Int64,
|
||||
processing_time Int64,
|
||||
insert_time Int64 ,
|
||||
address_type Int32,
|
||||
vsys_id Int32,
|
||||
client_ip String,
|
||||
client_port Int32,
|
||||
server_ip String,
|
||||
server_port Int32,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
dns_message_id Nullable(Int32),
|
||||
dns_qr Nullable(Int32),
|
||||
dns_opcode Nullable(Int32),
|
||||
dns_aa Nullable(Int32),
|
||||
dns_tc Nullable(Int32),
|
||||
dns_rd Nullable(Int32),
|
||||
dns_ra Nullable(Int32),
|
||||
dns_rcode Nullable(Int32),
|
||||
dns_qdcount Nullable(Int32),
|
||||
dns_ancount Nullable(Int32),
|
||||
dns_nscount Nullable(Int32),
|
||||
dns_arcount Nullable(Int32),
|
||||
dns_qname String,
|
||||
dns_qtype Nullable(Int32),
|
||||
dns_qclass Nullable(Int32),
|
||||
dns_cname String,
|
||||
dns_sub Nullable(Int32),
|
||||
dns_rr String,
|
||||
dns_response_latency_ms Nullable(Int32),
|
||||
http_url String,
|
||||
http_host String,
|
||||
http_request_line String,
|
||||
http_response_line String,
|
||||
http_request_body String,
|
||||
http_response_body String,
|
||||
http_proxy_flag Nullable(Int32),
|
||||
http_sequence Nullable(Int32),
|
||||
http_cookie String,
|
||||
http_referer String,
|
||||
http_user_agent String,
|
||||
http_request_content_length Nullable(Int64),
|
||||
http_request_content_type String,
|
||||
http_response_content_length Nullable(Int64),
|
||||
http_response_content_type String,
|
||||
http_set_cookie String,
|
||||
http_version String,
|
||||
http_status_code Nullable(Int32),
|
||||
http_response_latency_ms Nullable(Int32),
|
||||
http_session_duration_ms Nullable(Int32),
|
||||
http_action_file_size Nullable(Int64),
|
||||
mail_protocol_type String,
|
||||
mail_account String,
|
||||
mail_from_cmd String,
|
||||
mail_to_cmd String,
|
||||
mail_from String,
|
||||
mail_password String,
|
||||
mail_to String,
|
||||
mail_cc String,
|
||||
mail_bcc String,
|
||||
mail_subject String,
|
||||
mail_subject_charset String,
|
||||
mail_attachment_name String,
|
||||
mail_attachment_name_charset String,
|
||||
mail_starttls_flag Nullable(Int32),
|
||||
mail_eml_file String,
|
||||
sip_call_id String,
|
||||
sip_originator_description String,
|
||||
sip_responder_description String,
|
||||
sip_user_agent String,
|
||||
sip_server String,
|
||||
sip_originator_sdp_connect_ip String,
|
||||
sip_originator_sdp_media_port Nullable(Int32),
|
||||
sip_originator_sdp_media_type String,
|
||||
sip_originator_sdp_content String,
|
||||
sip_responder_sdp_connect_ip String,
|
||||
sip_responder_sdp_media_port Nullable(Int32),
|
||||
sip_responder_sdp_media_type String,
|
||||
sip_responder_sdp_content String,
|
||||
sip_duration_s Nullable(Int32),
|
||||
sip_bye String,
|
||||
sip_bye_reason String
|
||||
)
|
||||
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand());
|
||||
|
||||
|
||||
|
||||
alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1;
|
||||
alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1;
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
SELECT log_id,log_uuid, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
|
||||
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT vsys_id, recv_time, log_id,log_uuid, rule_id, rule_uuid, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip, source_country, sessions, session_rate, packets, packet_rate, bytes, bit_rate
|
||||
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id,log_uuid, decoded_as, session_id,session_uuid, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason
|
||||
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user