gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[CN-1704] location_subscriber库表调整

Browse Source
This commit is contained in:
LAPTOP-KOK0F0TH\hjc
2024-09-10 11:15:53 +08:00
parent a37ef55c93
commit 595b1f380a
9 changed files with 277 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql
View File

@@ -884,6 +884,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,
@@ -899,6 +901,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,
@@ -1486,3 +1490,221 @@ GROUP BY
ip,
l7_protocol,
port;
CREATE TABLE cyber_narrator_galaxy.match_indicator_local ON CLUSTER ck_cluster (
indicator_fields String,
indicator_values String,
match_num Int64,
reset Int64,
client_ip String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
server_ip String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
domain String,
app String,
match_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(match_time))
ORDER BY (match_id, match_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_cluster (
indicator_fields String,
indicator_values String,
match_num Int64,
reset Int64,
client_ip String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
server_ip String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
domain String,
app String,
match_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_indicator_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_threshold_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
threshold_value Float32,
metric_value Float32,
unit Int64 DEFAULT 1,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_cluster (
key_fields String,
key_values String,
threshold_value Float32,
metric_value Float32,
unit Int64 DEFAULT 1,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_threshold_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_sequence_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_sequence ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_sequence_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_unordered_sequence_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_unordered_sequence ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_unordered_sequence_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event_local ON CLUSTER ck_cluster(
event_id UInt64,
match_ids SimpleAggregateFunction(anyLast, String),
key_fields SimpleAggregateFunction(anyLast, String),
key_values SimpleAggregateFunction(anyLast, String),
rule_id SimpleAggregateFunction(anyLast, Int64),
rule_version SimpleAggregateFunction(anyLast, String),
rule_type SimpleAggregateFunction(anyLast, Int8),
is_builtin SimpleAggregateFunction(anyLast, Int8),
event_type SimpleAggregateFunction(anyLast, String),
event_name SimpleAggregateFunction(anyLast, String),
reset SimpleAggregateFunction(anyLast, Int64),
start_time SimpleAggregateFunction(min, Int64),
end_time SimpleAggregateFunction(max, Int64),
duration_s SimpleAggregateFunction(max, Int64),
status SimpleAggregateFunction(max, Int8)
)
ENGINE=AggregatingMergeTree ORDER BY event_id;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event ON CLUSTER ck_cluster(
event_id UInt64,
match_ids SimpleAggregateFunction(anyLast, String),
key_fields SimpleAggregateFunction(anyLast, String),
key_values SimpleAggregateFunction(anyLast, String),
rule_id SimpleAggregateFunction(anyLast, Int64),
rule_version SimpleAggregateFunction(anyLast, String),
rule_type SimpleAggregateFunction(anyLast, Int8),
is_builtin SimpleAggregateFunction(anyLast, Int8),
event_type SimpleAggregateFunction(anyLast, String),
event_name SimpleAggregateFunction(anyLast, String),
reset SimpleAggregateFunction(anyLast, Int64),
start_time SimpleAggregateFunction(min, Int64),
end_time SimpleAggregateFunction(max, Int64),
duration_s SimpleAggregateFunction(max, Int64),
status SimpleAggregateFunction(max, Int8)
)
ENGINE = Distributed('ck_cluster',
'cyber_narrator_galaxy',
'cn_event_local',
rand());

13
cyber_narrator/installation/clickhouse/cn_clickhouse_ddl_check.sql
View File

@@ -51,7 +51,18 @@ SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts,
FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state
FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
SELECT subscriber_id, imei, imsi, phone_number, apn, cell_id, cell_type, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT indicator_fields, indicator_values, match_num, reset, client_ip, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, server_ip, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, domain, app, match_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_indicator where match_time >= toUnixTimestamp('2030-01-01 00:00:00') AND match_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, threshold_value, metric_value, unit, reset, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_threshold where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_unordered_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT event_id, match_ids, key_fields, key_values, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, reset,start_time, end_time, duration_s, status
FROM cyber_narrator_galaxy.cn_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');

27
cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl.sql
View File

@@ -1306,6 +1306,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,
@@ -1321,6 +1323,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,
@@ -1336,6 +1340,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,
@@ -2225,9 +2231,10 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_q
CREATE TABLE cyber_narrator_galaxy.match_threshold_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
threshold_num Int64,
records_num Int64,
reset Int64,
threshold_value Float32,
metric_value Float32,
unit Int64 DEFAULT 1,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
@@ -2246,9 +2253,10 @@ ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_cluster (
key_fields String,
key_values String,
threshold_num Int64,
records_num Int64,
reset Int64,
threshold_value Float32,
metric_value Float32,
unit Int64 DEFAULT 1,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
@@ -2264,9 +2272,10 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_c
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_query (
key_fields String,
key_values String,
threshold_num Int64,
records_num Int64,
reset Int64,
threshold_value Float32,
metric_value Float32,
unit Int64 DEFAULT 1,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,

4
cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_24.04.sql
View File

@@ -884,6 +884,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,
@@ -899,6 +901,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,

2
cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_check_24.04.sql
View File

@@ -51,7 +51,7 @@ SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts,
FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state
FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
SELECT subscriber_id, imei, imsi, phone_number, apn, cell_id, cell_type, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

9
cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_upgrade_24.04.sql
View File

@@ -245,4 +245,11 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_cluste
domain_sketch String,
ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_type Int64 after apn;
ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_type Int64 after apn;
ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;
ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;

4
cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_24.08.sql
View File

@@ -884,6 +884,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,
@@ -899,6 +901,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
imsi String,
phone_number String,
apn String,
cell_id String,
cell_type Int64,
subscriber_longitude Nullable(Float64),
subscriber_latitude Nullable(Float64),
first_location String,

2
cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_check_24.08.sql
View File

@@ -51,7 +51,7 @@ SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts,
FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state
FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
SELECT subscriber_id, imei, imsi, phone_number, apn, cell_id, cell_type, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

8
cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_upgrade_24.08.sql
View File

@@ -1,3 +1,5 @@
set distributed_ddl_task_timeout = 180;
CREATE TABLE cyber_narrator_galaxy.match_indicator_local ON CLUSTER ck_cluster (
indicator_fields String,
indicator_values String,
@@ -217,4 +219,8 @@ ENGINE = Distributed('ck_cluster',
rand());
ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_type Int64 after apn;
ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_type Int64 after apn;
ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;
ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;