gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Clickhouse-dll.sql

Browse Source
This commit is contained in:
戚岱杰
2021-08-31 09:58:05 +00:00
parent 92485bec8b
commit d7ec6e64d8
1 changed files with 22 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
Clickhouse最新全量建表语句/Clickhouse-dll.sql
View File

@@ -1,13 +1,17 @@
CREATE DICTIONARY IF NOT EXISTS tsg_galaxy_v3.cdn on cluster ck_cluster ( create database IF NOT EXISTS tsg_galaxy_dll ON CLUSTER ck_cluster;
create database IF NOT EXISTS tsg_galaxy_dll ON CLUSTER ck_query;
CREATE DICTIONARY IF NOT EXISTS tsg_galaxy_dll.cdn on cluster ck_cluster (
cdn_id UInt64, cdn_id UInt64,
domain String, domain String,
cname String) PRIMARY KEY cdn_id SOURCE(MYSQL(PORT 3306 USER 'root' PASSWORD 'bifang!@#' REPLICA (HOST '192.168.44.71' PRIORITY 1) DB 'tsg-bifang' TABLE 'tsg_cdn_domain_info')) LIFETIME(MIN 300 MAX 400) LAYOUT(FLAT()); cname String) PRIMARY KEY cdn_id SOURCE(MYSQL(PORT 3306 USER 'root' PASSWORD 'bifang!@#' REPLICA (HOST '192.168.44.71' PRIORITY 1) DB 'tsg-bifang' TABLE 'tsg_cdn_domain_info')) LIFETIME(MIN 300 MAX 400) LAYOUT(FLAT());
create table IF NOT EXISTS tsg_galaxy_v3.cdn_dic on cluster ck_cluster (`cdn_id` UInt64, create table IF NOT EXISTS tsg_galaxy_dll.cdn_dic on cluster ck_cluster (`cdn_id` UInt64,
`domain` String, `domain` String,
`cname` String) Engine = Dictionary(tsg_galaxy_v3.cdn); `cname` String) Engine = Dictionary(tsg_galaxy_dll.cdn);
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local on cluster ck_cluster( CREATE TABLE IF NOT EXISTS tsg_galaxy_dll.security_website_domain_info_local on cluster ck_cluster(
stat_time Int64, stat_time Int64,
policy_id Int64, policy_id Int64,
domain String, domain String,
@@ -19,7 +23,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local on
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.security_website_domain_info_local ( CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_dll.security_website_domain_info_local_view on cluster ck_cluster TO tsg_galaxy_dll.security_website_domain_info_local (
stat_time UInt32, stat_time UInt32,
policy_id Int64, policy_id Int64,
domain String, domain String,
@@ -45,11 +49,11 @@ ip_list AggregateFunction(groupUniqArray,
'*', '*',
''))) AS san, ''))) AS san,
common_schema_type, common_schema_type,
common_server_port FROM tsg_galaxy_v3.security_event_log_local) AS sell INNER JOIN tsg_galaxy_v3.cdn_dic AS cc ON sell.san = cc.domain GROUP BY toStartOfDay(toDate(common_recv_time)), common_server_port FROM tsg_galaxy_dll.security_event_log_local) AS sell INNER JOIN tsg_galaxy_dll.cdn_dic AS cc ON sell.san = cc.domain GROUP BY toStartOfDay(toDate(common_recv_time)),
common_policy_id, common_policy_id,
http_domain; http_domain;
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local on cluster ck_cluster ( CREATE TABLE IF NOT EXISTS tsg_galaxy_dll.security_ip_info_local on cluster ck_cluster (
stat_time Int64, stat_time Int64,
policy_id Int64, policy_id Int64,
ip String, ip String,
@@ -58,7 +62,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local on cluster ck_c
) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,ip,stat_time) SETTINGS index_granularity = 8192; ) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,ip,stat_time) SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.security_ip_info_local AS CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_dll.security_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_dll.security_ip_info_local AS
SELECT SELECT
toUnixTimestamp(toStartOfDay(toDate(common_recv_time))) as stat_time, toUnixTimestamp(toStartOfDay(toDate(common_recv_time))) as stat_time,
common_policy_id as policy_id, common_policy_id as policy_id,
@@ -66,30 +70,30 @@ SELECT
groupUniqArrayState(http_domain) as domain_list , groupUniqArrayState(http_domain) as domain_list ,
groupUniqArrayState(common_server_port) as port_list groupUniqArrayState(common_server_port) as port_list
FROM FROM
tsg_galaxy_v3.security_event_log_local tsg_galaxy_dll.security_event_log_local
group BY group BY
toStartOfDay(toDate(common_recv_time)), toStartOfDay(toDate(common_recv_time)),
common_policy_id, common_policy_id,
common_server_ip; common_server_ip;
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info_local on cluster ck_cluster( CREATE TABLE IF NOT EXISTS tsg_galaxy_dll.proxy_ip_info_local on cluster ck_cluster(
stat_time Int64, stat_time Int64,
policy_id Int64, policy_id Int64,
ip_list AggregateFunction(groupUniqArray,String) ip_list AggregateFunction(groupUniqArray,String)
) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,stat_time) SETTINGS index_granularity = 8192; ) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,stat_time) SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.proxy_ip_info_local ( CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_dll.proxy_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_dll.proxy_ip_info_local (
stat_time UInt32, stat_time UInt32,
policy_id Int64, policy_id Int64,
ip_list AggregateFunction(groupUniqArray, ip_list AggregateFunction(groupUniqArray,
String)) AS SELECT toUnixTimestamp(toStartOfMinute(toDateTime(common_recv_time))) AS stat_time, String)) AS SELECT toUnixTimestamp(toStartOfMinute(toDateTime(common_recv_time))) AS stat_time,
common_policy_id AS policy_id, common_policy_id AS policy_id,
groupUniqArrayState(common_client_ip) AS ip_list FROM tsg_galaxy_v3.proxy_event_log_local GROUP BY toStartOfMinute(toDateTime(common_recv_time)), groupUniqArrayState(common_client_ip) AS ip_list FROM tsg_galaxy_dll.proxy_event_log_local GROUP BY toStartOfMinute(toDateTime(common_recv_time)),
common_policy_id; common_policy_id;
create table IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info on cluster ck_query create table IF NOT EXISTS tsg_galaxy_dll.security_website_domain_info on cluster ck_query
( (
stat_time Int64, stat_time Int64,
policy_id Int64, policy_id Int64,
@@ -99,20 +103,20 @@ create table IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info on cluster
protocol_type_list AggregateFunction(groupUniqArray,String), protocol_type_list AggregateFunction(groupUniqArray,String),
port_list AggregateFunction(groupUniqArray,Int64) port_list AggregateFunction(groupUniqArray,Int64)
) )
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_website_domain_info_local,rand()); ENGINE =Distributed(ck_cluster,tsg_galaxy_dll,security_website_domain_info_local,rand());
create table IF NOT EXISTS tsg_galaxy_v3.security_ip_info on cluster ck_query ( create table IF NOT EXISTS tsg_galaxy_dll.security_ip_info on cluster ck_query (
stat_time Int64, stat_time Int64,
policy_id Int64, policy_id Int64,
ip String, ip String,
domain_list AggregateFunction(groupUniqArray,String), domain_list AggregateFunction(groupUniqArray,String),
port_list AggregateFunction(groupUniqArray,Int64) port_list AggregateFunction(groupUniqArray,Int64)
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_ip_info_local,rand()); ) ENGINE =Distributed(ck_cluster,tsg_galaxy_dll,security_ip_info_local,rand());
create table IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info on cluster ck_query ( create table IF NOT EXISTS tsg_galaxy_dll.proxy_ip_info on cluster ck_query (
stat_time Int64, stat_time Int64,
policy_id Int64, policy_id Int64,
ip_list AggregateFunction(groupUniqArray,String) ip_list AggregateFunction(groupUniqArray,String)
) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_ip_info_local,rand()); ) ENGINE =Distributed(ck_cluster,tsg_galaxy_dll,proxy_ip_info_local,rand());