diff --git a/CN发布版本更新记录/CN-23.09/clickhouse/update-23.09-ck.sql b/CN发布版本更新记录/CN-23.09/clickhouse/update-23.09-ck.sql index fabd6f2..c153339 100644 --- a/CN发布版本更新记录/CN-23.09/clickhouse/update-23.09-ck.sql +++ b/CN发布版本更新记录/CN-23.09/clickhouse/update-23.09-ck.sql @@ -1,5 +1,36 @@ set distributed_ddl_task_timeout = 180; +drop table IF EXISTS cyber_narrator_galaxy.metric_idc_renter_local on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.metric_idc_renter on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.metric_idc_renter on cluster ck_query; + +drop table IF EXISTS cyber_narrator_galaxy.metric_ssl_sni_local on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.metric_ssl_sni on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.metric_ssl_sni on cluster ck_query; + +drop table IF EXISTS cyber_narrator_galaxy.metric_http_host_local on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.metric_http_host on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.metric_http_host on cluster ck_query; + + +drop table IF EXISTS cyber_narrator_galaxy.metric_region_local on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.metric_region on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.metric_region on cluster ck_query; + + +drop table IF EXISTS cyber_narrator_galaxy.cn_dynamic_info_relation_local on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.cn_dynamic_info_relation on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.cn_dynamic_info_relation on cluster ck_query; + +drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation_local on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_cluster; +drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_query; + +drop view IF EXISTS cyber_narrator_galaxy.cn_dynamic_info_relation_view on cluster ck_cluster; +drop view IF EXISTS cyber_narrator_galaxy.cn_entity_relation_view on cluster ck_cluster; + + + ALTER table cyber_narrator_galaxy.session_record_cn_local on cluster ck_cluster add column IF NOT EXISTS common_flags UInt64 after common_recv_time , add column IF NOT EXISTS common_log_id UInt64 after common_recv_time , add column IF NOT EXISTS common_app_full_path String after common_app_label , add column IF NOT EXISTS domain_sld String after domain; ALTER table cyber_narrator_galaxy.session_record_cn on cluster ck_cluster add column IF NOT EXISTS common_flags UInt64 after common_recv_time ,add column IF NOT EXISTS common_log_id UInt64 after common_recv_time , add column IF NOT EXISTS common_app_full_path String after common_app_label , add column IF NOT EXISTS domain_sld String after domain; ALTER table cyber_narrator_galaxy.session_record_cn on cluster ck_query add column IF NOT EXISTS common_flags UInt64 after common_recv_time ,add column IF NOT EXISTS common_log_id UInt64 after common_recv_time , add column IF NOT EXISTS common_app_full_path String after common_app_label , add column IF NOT EXISTS domain_sld String after domain; @@ -38,23 +69,6 @@ ALTER table cyber_narrator_galaxy.metric_domain on cluster ck_cluster add colum ALTER table cyber_narrator_galaxy.metric_domain on cluster ck_query add column IF NOT EXISTS domain_sld String after domain; -drop table IF EXISTS cyber_narrator_galaxy.metric_idc_renter_local on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.metric_idc_renter on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.metric_idc_renter on cluster ck_query; - -drop table IF EXISTS cyber_narrator_galaxy.metric_ssl_sni_local on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.metric_ssl_sni on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.metric_ssl_sni on cluster ck_query; - -drop table IF EXISTS cyber_narrator_galaxy.metric_http_host_local on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.metric_http_host on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.metric_http_host on cluster ck_query; - - -drop table IF EXISTS cyber_narrator_galaxy.metric_region_local on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.metric_region on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.metric_region on cluster ck_query; - CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_region_local ON CLUSTER ck_cluster ( country_region String, @@ -240,19 +254,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute ON - -drop table IF EXISTS cyber_narrator_galaxy.cn_dynamic_info_relation_local on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.cn_dynamic_info_relation on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.cn_dynamic_info_relation on cluster ck_query; - -drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation_local on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_cluster; -drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_query; - -drop view IF EXISTS cyber_narrator_galaxy.cn_dynamic_info_relation_view on cluster ck_cluster; -drop view IF EXISTS cyber_narrator_galaxy.cn_entity_relation_view on cluster ck_cluster; - - CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_entity_relation_local on cluster ck_cluster ( diff --git a/CN发布版本更新记录/CN-23.10/clickhouse/update-23.10-ck.sql b/CN发布版本更新记录/CN-23.10/clickhouse/update-23.10-ck.sql index 1ecf78b..0b87201 100644 --- a/CN发布版本更新记录/CN-23.10/clickhouse/update-23.10-ck.sql +++ b/CN发布版本更新记录/CN-23.10/clickhouse/update-23.10-ck.sql @@ -39,3 +39,105 @@ ALTER table cyber_narrator_galaxy.metric_ip on cluster ck_query add column IF NO ALTER table cyber_narrator_galaxy.metric_ip_local on cluster ck_cluster add column IF NOT EXISTS bidirectional_sessions Int64 after random_looking_sessions; ALTER table cyber_narrator_galaxy.metric_ip on cluster ck_cluster add column IF NOT EXISTS bidirectional_sessions Int64 after random_looking_sessions; ALTER table cyber_narrator_galaxy.metric_ip on cluster ck_query add column IF NOT EXISTS bidirectional_sessions Int64 after random_looking_sessions; + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event_local ON CLUSTER ck_cluster +( + + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + offender_ip String, + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = MergeTree +ORDER BY event_id +TTL toDateTime(end_time) + toIntervalSecond(2592000), + toDateTime(end_time) + toIntervalSecond(1) GROUP BY event_id + SET + status = anyLast(status), + is_builtin = anyLast(is_builtin), + rule_type = anyLast(rule_type), + victim_ip = anyLast(victim_ip), + offender_ip = anyLast(offender_ip), + event_info = anyLast(event_info), + event_key = anyLast(event_key), + severity = anyLast(severity), + event_type = anyLast(event_type), + duration_s = anyLast(duration_s), + event_name = anyLast(event_name), + app = anyLast(app), + domain = anyLast(domain), + rule_id = anyLast(rule_id), + start_time = anyLast(start_time), + end_time = max(end_time), + match_times = anyLast(match_times); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_cluster +( + + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + offender_ip String, + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'cn_security_event_local', + rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_query +( + + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + offender_ip String, + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'cn_security_event_local', + rand()); + diff --git a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql index 69331bb..8f16d7d 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_CN_建表语句.sql @@ -1694,4 +1694,108 @@ FROM cyber_narrator_galaxy.metric_ip_dynamic_attribute_local AS c1 GROUP BY ip, l7_protocol, - port; \ No newline at end of file + port; + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event_local ON CLUSTER ck_cluster +( + + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + offender_ip String, + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = MergeTree +ORDER BY event_id +TTL toDateTime(end_time) + toIntervalSecond(2592000), + toDateTime(end_time) + toIntervalSecond(1) GROUP BY event_id + SET + status = anyLast(status), + is_builtin = anyLast(is_builtin), + rule_type = anyLast(rule_type), + victim_ip = anyLast(victim_ip), + offender_ip = anyLast(offender_ip), + event_info = anyLast(event_info), + event_key = anyLast(event_key), + severity = anyLast(severity), + event_type = anyLast(event_type), + duration_s = anyLast(duration_s), + event_name = anyLast(event_name), + app = anyLast(app), + domain = anyLast(domain), + rule_id = anyLast(rule_id), + start_time = anyLast(start_time), + end_time = max(end_time), + match_times = anyLast(match_times); + + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_cluster +( + + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + offender_ip String, + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'cn_security_event_local', + rand()); + +CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_security_event ON CLUSTER ck_query +( + + status Int64, + is_builtin Int64, + rule_type String, + victim_ip String, + offender_ip String, + event_info String, + event_key String, + severity String, + event_type String, + duration_s Int64, + event_name String, + app String, + domain String, + event_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + match_times Int64 +) +ENGINE = Distributed('ck_cluster', + 'cyber_narrator_galaxy', + 'cn_security_event_local', + rand()); + + + +