From 063b4520c921f3a19c04d35f2400d48766f6c560 Mon Sep 17 00:00:00 2001 From: houjinchuan Date: Wed, 28 Dec 2022 16:57:10 +0800 Subject: [PATCH] =?UTF-8?q?tsg=20ck=2023.01=E5=A2=9E=E5=8A=A0=E5=AD=97?= =?UTF-8?q?=E6=AE=B5common=5Fserver=5Ffqdn=EF=BC=8Ccommon=5Fserver=5Fdomai?= =?UTF-8?q?n=EF=BC=8C=E5=A2=9E=E5=8A=A0=E5=AD=90=E8=A1=A8session=5Frecord?= =?UTF-8?q?=5Fcommon=5Fserver=5Fdomain=E5=92=8C=E5=AF=B9=E5=BA=94=E7=9A=84?= =?UTF-8?q?=E7=89=A9=E5=8C=96=E8=A7=86=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Clickhouse_TSG_建表语句_NEW.sql | 145 +++++++++++++++- .../Clickhouse_TSG_建表语句_OLD.sql | 151 +++++++++++++++- .../TSG-23.01/clickhouse/update-23.01-ck.sql | 163 +++++++++++++++++- 3 files changed, 443 insertions(+), 16 deletions(-) diff --git a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_NEW.sql b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_NEW.sql index 9ff4e6b..845e58e 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_NEW.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_NEW.sql @@ -154,6 +154,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_local ON CLUSTER ck_cluster common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -261,6 +263,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_cluster( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -364,6 +368,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_query( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -468,6 +474,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record_local ON CLUSTER common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -724,6 +732,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cl common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -979,6 +989,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_qu common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1234,6 +1246,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_ common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1395,6 +1409,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluste common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1553,6 +1569,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query( common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1711,6 +1729,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local ON CLUSTER ck_cluster common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1826,6 +1846,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1939,6 +1961,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2052,6 +2076,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event_local ON CLUST common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2155,6 +2181,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_ common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2256,6 +2284,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_ common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2357,6 +2387,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local ON CLUSTER ck_clus common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2614,6 +2646,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2869,6 +2903,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3125,6 +3161,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local ON CLUSTER ck_cluster common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3296,6 +3334,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3464,6 +3504,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3632,6 +3674,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local ON CLUSTER ck_clus common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3889,6 +3933,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster ( common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4144,6 +4190,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query ( common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4399,6 +4447,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_local ON CLUSTER ck_clust common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4524,6 +4574,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_cluster ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4646,6 +4698,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_query ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4738,6 +4792,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip_local O common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4762,6 +4817,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4775,7 +4831,6 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); - CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_query( common_log_id UInt64, common_recv_time Int64, @@ -4785,6 +4840,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4808,6 +4864,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip_local O common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4832,6 +4889,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4845,7 +4903,6 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); - CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_query( common_log_id UInt64, common_recv_time Int64, @@ -4855,6 +4912,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4868,6 +4926,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLUSTER ck_cluster( common_log_id UInt64, common_recv_time Int64, @@ -4877,6 +4936,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLU common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4901,6 +4961,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4923,6 +4984,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4937,9 +4999,82 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_domain,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; diff --git a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_OLD.sql b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_OLD.sql index d6519df..d1486ee 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_OLD.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句_OLD.sql @@ -61,7 +61,7 @@ ENGINE = Distributed('ck_cluster', CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local ON CLUSTER ck_cluster( -log_id UInt64, + log_id UInt64, start_time Int64, end_time Int64, attack_type String, @@ -156,6 +156,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_local ON CLUSTER ck_cluster common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -225,7 +227,7 @@ ORDER BY (common_log_id,common_data_center,common_recv_time,common_vsys_id); CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_cluster( - common_recv_time Int64, + common_recv_time Int64, common_log_id UInt64, common_stream_trace_id UInt64, common_direction Nullable(Int64), @@ -264,6 +266,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_cluster( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -367,6 +371,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record ON CLUSTER ck_query( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -471,6 +477,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record_local ON CLUSTER common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -728,6 +736,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_cl common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -983,6 +993,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.interim_session_record ON CLUSTER ck_qu common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1238,6 +1250,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local ON CLUSTER ck_ common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1400,6 +1414,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_cluste common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1558,6 +1574,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record ON CLUSTER ck_query( common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1716,6 +1734,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local ON CLUSTER ck_cluster common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1832,6 +1852,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_cluster( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -1945,6 +1967,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record ON CLUSTER ck_query( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2058,6 +2082,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event_local ON CLUST common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2162,6 +2188,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_ common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2263,6 +2291,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_event ON CLUSTER ck_ common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2364,6 +2394,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local ON CLUSTER ck_clus common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2622,6 +2654,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_cluster ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -2877,6 +2911,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_event ON CLUSTER ck_query ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3133,6 +3169,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local ON CLUSTER ck_cluster common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3305,6 +3343,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_cluster ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3473,6 +3513,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.proxy_event ON CLUSTER ck_query ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3641,6 +3683,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local ON CLUSTER ck_clus common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -3899,6 +3943,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_cluster ( common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4154,6 +4200,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.session_record ON CLUSTER ck_query ( common_server_port Int64, common_server_location LowCardinality(String), common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4409,6 +4457,8 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_local ON CLUSTER ck_clust common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4535,6 +4585,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_cluster ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4657,6 +4709,8 @@ create table IF NOT EXISTS tsg_galaxy_v3.radius_record ON CLUSTER ck_query ( common_server_port Int64, common_server_location String, common_server_asn String, + common_server_fqdn String, + common_server_domain String, common_app_id String, common_app_full_path String, common_userdefine_app_name String, @@ -4749,6 +4803,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip_local O common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4773,6 +4828,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4786,7 +4842,6 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_client_ip_local,rand()); - CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUSTER ck_query( common_log_id UInt64, common_recv_time Int64, @@ -4796,6 +4851,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_client_ip ON CLUS common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4819,6 +4875,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip_local O common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4843,6 +4900,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4856,7 +4914,6 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); - CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUSTER ck_query( common_log_id UInt64, common_recv_time Int64, @@ -4866,6 +4923,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4879,6 +4937,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_ip ON CLUS ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_ip_local,rand()); + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLUSTER ck_cluster( common_log_id UInt64, common_recv_time Int64, @@ -4888,6 +4947,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain_local ON CLU common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4912,6 +4972,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4934,6 +4995,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c common_entrance_id Int64, common_subscriber_id String, common_stream_trace_id UInt64, + common_server_domain String, http_domain String, ssl_sni String, common_schema_type LowCardinality(String), @@ -4948,9 +5010,82 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_http_domain ON CLUSTER c -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_domain,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; @@ -5007,7 +5142,7 @@ common_recv_time Int64, common_entrance_id Int64, common_device_id String, common_egress_link_id Nullable(Int64), - common_ingress_link_id Nullable(Int64), +common_ingress_link_id Nullable(Int64), common_policy_id Int64, common_user_region String, ad_method String, diff --git a/TSG发布版本更新记录/TSG-23.01/clickhouse/update-23.01-ck.sql b/TSG发布版本更新记录/TSG-23.01/clickhouse/update-23.01-ck.sql index 86dacd5..61241eb 100644 --- a/TSG发布版本更新记录/TSG-23.01/clickhouse/update-23.01-ck.sql +++ b/TSG发布版本更新记录/TSG-23.01/clickhouse/update-23.01-ck.sql @@ -37,6 +37,149 @@ ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add col ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_domain,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + + ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; @@ -63,10 +206,24 @@ ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add c ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + +ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + +ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + + drop view IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster; drop view IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster; drop view IF EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; -CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; \ No newline at end of file +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; \ No newline at end of file