gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
zhanghongqing
2024-08-05 10:31:43 +08:00
parent b5c33dd722 f6abcd85ac
commit 2f1b1f5929
21 changed files with 8610 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql
View File

@@ -855,8 +855,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE
CREATE TABLE cyber_narrator_galaxy.metric_tag_local
(
CREATE TABLE cyber_narrator_galaxy.metric_tag_local ON CLUSTER ck_cluster (
tag String,
stat_time Int64,
ip_sketch String,

259
cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl.sql
View File

@@ -1269,8 +1269,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_subscriber_app ON CLUSTE
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_subscriber_app_local', rand());
CREATE TABLE cyber_narrator_galaxy.metric_tag_local
(
CREATE TABLE cyber_narrator_galaxy.metric_tag_local ON CLUSTER ck_cluster (
tag String,
stat_time Int64,
ip_sketch String,
@@ -2127,3 +2126,259 @@ GROUP BY
ip,
l7_protocol,
port;
CREATE TABLE cyber_narrator_galaxy.match_indicator_local ON CLUSTER ck_cluster (
indicator_fields String,
indicator_values String,
match_num Int64,
reset Int64,
client_ip String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
server_ip String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
domain String,
app String,
match_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(match_time))
ORDER BY (match_id, match_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_cluster (
indicator_fields String,
indicator_values String,
match_num Int64,
reset Int64,
client_ip String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
server_ip String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
domain String,
app String,
match_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_indicator_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_query (
indicator_fields String,
indicator_values String,
match_num Int64,
reset Int64,
client_ip String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
server_ip String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
domain String,
app String,
match_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_indicator_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_threshold_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
threshold_num Int64,
records_num Int64,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_cluster (
key_fields String,
key_values String,
threshold_num Int64,
records_num Int64,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_threshold_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_query (
key_fields String,
key_values String,
threshold_num Int64,
records_num Int64,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_threshold_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_sequence_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_sequence ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_sequence_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_sequence ON CLUSTER ck_query (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_sequence_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_unordered_sequence_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_unordered_sequence ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_unordered_sequence_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_unordered_sequence ON CLUSTER ck_query (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_unordered_sequence_local', rand());

9
cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl_check.sql
View File

@@ -55,3 +55,12 @@ SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subsc
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT indicator_fields, indicator_values, match_num, reset, client_ip, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, server_ip, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, domain, app, match_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_indicator where match_time >= toUnixTimestamp('2030-01-01 00:00:00') AND match_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, threshold_num, records_num, reset, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_threshold where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_unordered_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');

1488
cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_24.04.sql Normal file
View File

File diff suppressed because it is too large Load Diff

47
cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_check_24.04.sql
View File

@@ -1,14 +1,14 @@
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_domain_ip_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, create_time, update_time
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT status, is_builtin, rule_type, victim_ip, offender_ip, event_info, event_key, severity, event_type, duration_s, event_name, app, `domain`, event_id, rule_id, start_time, end_time, match_times
FROM cyber_narrator_galaxy.cn_security_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name ,domain ,ip ,ip_country_region ,ip_super_admin_area ,ip_admin_area ,ip_asn ,ip_isp ,domain_category_name ,domain_category_group ,app_category ,app_subcategory ,entity_tags ,stat_time
FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, stat_time
FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name, subscriber_id, imei, imsi, phone_number, apn, app_category, app_subcategory, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_subscriber_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name ,subscriber_id ,imei ,imsi ,phone_number ,apn ,app_category ,app_subcategory ,stat_time
SELECT app_name , subscriber_id , imei , imsi , phone_number , apn , app_category , app_subcategory , stat_time
FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
@@ -16,5 +16,42 @@ SELECT app_name,subscriber_id,imei,imsi,phone_number,apn,app_category,app_subcat
FROM cyber_narrator_galaxy.session_relation_subscriber_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, create_time, update_time
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app, app_category, app_subcategory, app_company, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_application where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT asn, isp, side, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_asn where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT qname, qname_sld, qname_tld, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_qname where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT qtype, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_qtype where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rcode, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rcode where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_a, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_a where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_aaaa, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_aaaa where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_cname, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_cname where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT server_ip, server_country_region, server_super_admin_area, server_admin_area, server_isp, server_role, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_server_ip where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT `domain`, domain_sld, domain_category_name, domain_category_group, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_domain where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, side, `zone`, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms, asymmetric_sessions, bulky_sessions, cbr_streaming_sessions, download_sessions, interactive_sessions, pseudo_unidirectional_sessions, streaming_sessions, unidirectional_sessions, random_looking_sessions, bidirectional_sessions
FROM cyber_narrator_galaxy.metric_ip where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, stat_time
FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT client_country_region, client_super_admin_area, client_admin_area, client_zone, server_country_region, server_super_admin_area, server_admin_area, server_zone, out_link_id, in_link_id, out_link_direction, in_link_direction, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_link where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT l7_protocol, server_port, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_protocol where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT country_region, super_admin_area, admin_area, side, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_region where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state
FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

87
cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_upgrade_24.04.sql
View File

@@ -2,7 +2,6 @@ set distributed_ddl_task_timeout = 180;
ALTER table cyber_narrator_galaxy.location_subscriber_local on cluster ck_cluster add column IF NOT EXISTS data_source String after third_location;
ALTER table cyber_narrator_galaxy.location_subscriber on cluster ck_cluster add column IF NOT EXISTS data_source String after third_location;
ALTER table cyber_narrator_galaxy.location_subscriber on cluster ck_query add column IF NOT EXISTS data_source String after third_location;
drop view IF EXISTS cyber_narrator_galaxy.cn_entity_relation_view ON CLUSTER ck_cluster;
drop view IF EXISTS cyber_narrator_galaxy.cn_ip_dynamic_attribute_view ON CLUSTER ck_cluster;
@@ -10,11 +9,8 @@ drop view IF EXISTS cyber_narrator_galaxy.cn_entity_relation_view_metric ON CLUS
drop view IF EXISTS cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric ON CLUSTER ck_cluster;
drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_cluster;
drop table IF EXISTS cyber_narrator_galaxy.cn_entity_relation on cluster ck_query;
drop table IF EXISTS cyber_narrator_galaxy.metric_relation on cluster ck_cluster;
drop table IF EXISTS cyber_narrator_galaxy.metric_relation on cluster ck_query;
drop table IF EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute on cluster ck_cluster;
drop table IF EXISTS cyber_narrator_galaxy.metric_ip_dynamic_attribute on cluster ck_query;
RENAME TABLE cyber_narrator_galaxy.cn_entity_relation_local TO cyber_narrator_galaxy.session_relation_domain_ip_app_local ON CLUSTER ck_cluster;
@@ -60,19 +56,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app
update_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_subscriber_app ON CLUSTER ck_query (
app_name String,
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
app_category String,
app_subcategory String,
create_time Int64,
update_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'session_relation_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app_local ON CLUSTER ck_cluster (
app_name String,
@@ -98,42 +81,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_subscriber_app ON CLUSTER ck_query (
app_name String,
subscriber_id String,
imei String,
imsi String,
phone_number String,
apn String,
app_category String,
app_subcategory String,
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_subscriber_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_query
(
app_name String,
domain String,
ip String,
ip_country_region String,
ip_super_admin_area String,
ip_admin_area String,
ip_asn String,
ip_isp String,
domain_category_name String,
domain_category_group String,
app_category String,
app_subcategory String,
entity_tags Array(String),
create_time Int64,
update_time Int64
)
ENGINE = Distributed('ck_cluster',
'cyber_narrator_galaxy',
'session_relation_domain_ip_app_local',
rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.session_relation_domain_ip_app on cluster ck_cluster
(
app_name String,
@@ -174,22 +123,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_session_relation_domain_ip_app ON CLUSTER ck_query (
app_name String,
domain String,
ip String,
ip_country_region String,
ip_super_admin_area String,
ip_admin_area String,
ip_asn String,
ip_isp String,
domain_category_name String,
domain_category_group String,
app_category String,
app_subcategory String,
entity_tags Array(String),
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_session_relation_domain_ip_app_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_cluster (
ip String,
@@ -198,12 +131,6 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute ON CLUSTER ck_query (
ip String,
l7_protocol String,
port Int64,
stat_time Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'raw_cn_ip_dynamic_attribute_local', rand());
CREATE MATERIALIZED VIEW if not exists cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric on cluster ck_cluster
@@ -299,8 +226,7 @@ GROUP BY
port;
CREATE TABLE cyber_narrator_galaxy.metric_tag_local
(
CREATE TABLE cyber_narrator_galaxy.metric_tag_local ON CLUSTER ck_cluster (
tag String,
stat_time Int64,
ip_sketch String,
@@ -319,13 +245,4 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_cluste
domain_sketch String,
ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_query (
tag String,
stat_time Int64,
ip_sketch String,
domain_sketch String,
ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());

0
cyber_narrator/upgrade/2024/CN-24.04/start_all.sh → cyber_narrator/upgrade/2024/CN-24.04/gohangout/start_all.sh
View File

1702
cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_24.08.sql Normal file
View File

File diff suppressed because it is too large Load Diff

68
cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_check_24.08.sql Normal file
View File

@@ -0,0 +1,68 @@
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_domain_ip_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, create_time, update_time
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT status, is_builtin, rule_type, victim_ip, offender_ip, event_info, event_key, severity, event_type, duration_s, event_name, app, `domain`, event_id, rule_id, start_time, end_time, match_times
FROM cyber_narrator_galaxy.cn_security_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name ,domain ,ip ,ip_country_region ,ip_super_admin_area ,ip_admin_area ,ip_asn ,ip_isp ,domain_category_name ,domain_category_group ,app_category ,app_subcategory ,entity_tags ,stat_time
FROM cyber_narrator_galaxy.raw_session_relation_domain_ip_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name, subscriber_id, imei, imsi, phone_number, apn, app_category, app_subcategory, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_subscriber_app where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name , subscriber_id , imei , imsi , phone_number , apn , app_category , app_subcategory , stat_time
FROM cyber_narrator_galaxy.raw_session_relation_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name, `domain`, ip, ip_country_region, ip_super_admin_area, ip_admin_area, ip_asn, ip_isp, domain_category_name, domain_category_group, app_category, app_subcategory, entity_tags, create_time, update_time
FROM cyber_narrator_galaxy.session_relation_domain_ip_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app_name,subscriber_id,imei,imsi,phone_number,apn,app_category,app_subcategory,create_time,update_time
FROM cyber_narrator_galaxy.session_relation_subscriber_app_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, create_time, update_time
FROM cyber_narrator_galaxy.cn_ip_dynamic_attribute_view_metric where create_time >= toUnixTimestamp('2030-01-01 00:00:00') AND create_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT app, app_category, app_subcategory, app_company, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_application where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT asn, isp, side, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_asn where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT qname, qname_sld, qname_tld, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_qname where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT qtype, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_qtype where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rcode, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rcode where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_a, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_a where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_aaaa, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_aaaa where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT rr_cname, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_rr_cname where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT server_ip, server_country_region, server_super_admin_area, server_admin_area, server_isp, server_role, stat_time, query_num, traffic_inbound_byte, traffic_outbound_byte, internal_query_num, external_query_num, avg_response_latency_ms
FROM cyber_narrator_galaxy.metric_dns_server_ip where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT `domain`, domain_sld, domain_category_name, domain_category_group, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_domain where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, side, `zone`, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms, asymmetric_sessions, bulky_sessions, cbr_streaming_sessions, download_sessions, interactive_sessions, pseudo_unidirectional_sessions, streaming_sessions, unidirectional_sessions, random_looking_sessions, bidirectional_sessions
FROM cyber_narrator_galaxy.metric_ip where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT ip, l7_protocol, port, stat_time
FROM cyber_narrator_galaxy.raw_cn_ip_dynamic_attribute where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT client_country_region, client_super_admin_area, client_admin_area, client_zone, server_country_region, server_super_admin_area, server_admin_area, server_zone, out_link_id, in_link_id, out_link_direction, in_link_direction, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_link where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT l7_protocol, server_port, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_protocol where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT country_region, super_admin_area, admin_area, side, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_region where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, traffic_inbound_byte, traffic_inbound_pkt, traffic_outbound_byte, traffic_outbound_pkt, traffic_internal_byte, traffic_internal_pkt, traffic_through_byte, traffic_through_pkt, tcp_c2s_lost_bytes_ratio, tcp_s2c_lost_bytes_ratio, tcp_lost_bytes_ratio, tcp_c2s_o3_pkts_ratio, tcp_s2c_o3_pkts_ratio, tcp_o3_pkts_ratio, tcp_c2s_rtx_bytes_ratio, tcp_s2c_rtx_bytes_ratio, tcp_rtx_bytes_ratio, tcp_c2s_rtx_pkts_ratio, tcp_s2c_rtx_pkts_ratio, tcp_rtx_pkts_ratio, avg_tcp_rtt_ms, avg_http_response_latency_ms, avg_ssl_handshake_latency_ms
FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state
FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source, stat_time
FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT indicator_fields, indicator_values, match_num, reset, client_ip, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, server_ip, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, domain, app, match_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_indicator where match_time >= toUnixTimestamp('2030-01-01 00:00:00') AND match_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, threshold_num, records_num, reset, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_threshold where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
FROM cyber_narrator_galaxy.match_unordered_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT event_id, match_ids, key_fields, key_values, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, start_time, end_time, duration_s, status
FROM cyber_narrator_galaxy.cn_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');

213
cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_upgrade_24.08.sql Normal file
View File

@@ -0,0 +1,213 @@
CREATE TABLE cyber_narrator_galaxy.match_indicator_local ON CLUSTER ck_cluster (
indicator_fields String,
indicator_values String,
match_num Int64,
reset Int64,
client_ip String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
server_ip String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
domain String,
app String,
match_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(match_time))
ORDER BY (match_id, match_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_cluster (
indicator_fields String,
indicator_values String,
match_num Int64,
reset Int64,
client_ip String,
client_country_region String,
client_super_admin_area String,
client_admin_area String,
client_longitude Nullable(Float64),
client_latitude Nullable(Float64),
server_ip String,
server_country_region String,
server_super_admin_area String,
server_admin_area String,
server_longitude Nullable(Float64),
server_latitude Nullable(Float64),
domain String,
app String,
match_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_indicator_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_threshold_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
threshold_num Int64,
records_num Int64,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_cluster (
key_fields String,
key_values String,
threshold_num Int64,
records_num Int64,
reset Int64,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_threshold_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_sequence_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_sequence ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_sequence_local', rand());
CREATE TABLE cyber_narrator_galaxy.match_unordered_sequence_local ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(toDate(start_time))
ORDER BY (match_id, start_time);
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_unordered_sequence ON CLUSTER ck_cluster (
key_fields String,
key_values String,
event_info String,
start_time Int64,
end_time Int64,
match_id UInt64,
rule_id UInt64,
rule_version String,
rule_type String,
is_builtin Int64,
event_type String,
event_name String,
severity Int64
) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_unordered_sequence_local', rand());
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event_local ON CLUSTER ck_cluster(
event_id UInt64,
match_ids SimpleAggregateFunction(anyLast, String),
key_fields SimpleAggregateFunction(anyLast, String),
key_values SimpleAggregateFunction(anyLast, String),
rule_id SimpleAggregateFunction(anyLast, Int64),
rule_version SimpleAggregateFunction(anyLast, String),
rule_type SimpleAggregateFunction(anyLast, Int8),
is_builtin SimpleAggregateFunction(anyLast, Int8),
event_type SimpleAggregateFunction(anyLast, String),
event_name SimpleAggregateFunction(anyLast, String),
start_time SimpleAggregateFunction(min, Int64),
end_time SimpleAggregateFunction(max, Int64),
duration_s SimpleAggregateFunction(max, Int64),
status SimpleAggregateFunction(max, Int8)
)
ENGINE=AggregatingMergeTree ORDER BY event_id;
CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event ON CLUSTER ck_cluster(
event_id UInt64,
match_ids SimpleAggregateFunction(anyLast, String),
key_fields SimpleAggregateFunction(anyLast, String),
key_values SimpleAggregateFunction(anyLast, String),
rule_id SimpleAggregateFunction(anyLast, Int64),
rule_version SimpleAggregateFunction(anyLast, String),
rule_type SimpleAggregateFunction(anyLast, Int8),
is_builtin SimpleAggregateFunction(anyLast, Int8),
event_type SimpleAggregateFunction(anyLast, String),
event_name SimpleAggregateFunction(anyLast, String),
start_time SimpleAggregateFunction(min, Int64),
end_time SimpleAggregateFunction(max, Int64),
duration_s SimpleAggregateFunction(max, Int64),
status SimpleAggregateFunction(max, Int8)
)
ENGINE = Distributed('ck_cluster',
'cyber_narrator_galaxy',
'cn_event_local',
rand());

37
tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl.sql
View File

@@ -137,6 +137,7 @@ proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -151,6 +152,7 @@ imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -160,6 +162,7 @@ server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
@@ -383,6 +386,7 @@ proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -397,6 +401,7 @@ imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -406,6 +411,7 @@ server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
@@ -627,6 +633,7 @@ proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -641,6 +648,7 @@ imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -650,6 +658,7 @@ server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
@@ -872,6 +881,7 @@ proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -886,6 +896,7 @@ imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -895,6 +906,7 @@ server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
@@ -1116,6 +1128,7 @@ proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -1130,6 +1143,7 @@ imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -1139,6 +1153,7 @@ server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
@@ -1361,6 +1376,7 @@ proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -1375,6 +1391,7 @@ imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -1384,6 +1401,7 @@ server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
@@ -1939,6 +1957,7 @@ proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -1953,6 +1972,7 @@ imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -1962,6 +1982,7 @@ server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
@@ -2104,6 +2125,7 @@ proxy_intercept_error String,
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -2118,6 +2140,7 @@ imsi String,
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -2127,6 +2150,7 @@ server_administrative_area String,
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String, 
app LowCardinality(String),
@@ -2218,7 +2242,6 @@ tunnel_endpoint_b_desc String
ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand());
-- tsg_galaxy_v3.security_event_materialized_view
CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster
TO tsg_galaxy_v3.security_event_local
@@ -2271,6 +2294,7 @@ TO tsg_galaxy_v3.security_event_local
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -2285,6 +2309,7 @@ TO tsg_galaxy_v3.security_event_local
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -2294,6 +2319,7 @@ TO tsg_galaxy_v3.security_event_local
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String,
app LowCardinality(String),
@@ -2513,6 +2539,7 @@ SELECT
monitor_mirrored_pkts,
monitor_mirrored_bytes,
client_ip,
client_ip_tags,
client_port,
client_os_desc,
client_geolocation,
@@ -2527,6 +2554,7 @@ SELECT
phone_number,
apn,
server_ip,
server_ip_tags,
server_port,
server_os_desc,
server_geolocation,
@@ -2536,6 +2564,7 @@ SELECT
server_sub_administrative_area,
server_asn,
server_fqdn,
server_fqdn_tags,
server_domain,
app_transition,
app,
@@ -2760,6 +2789,7 @@ TO tsg_galaxy_v3.monitor_event_local
monitor_mirrored_pkts Nullable(Int32),
monitor_mirrored_bytes Nullable(Int32),
client_ip String,
client_ip_tags Array(String),
client_port Int32,
client_os_desc String,
client_geolocation LowCardinality(String),
@@ -2774,6 +2804,7 @@ TO tsg_galaxy_v3.monitor_event_local
phone_number String,
apn String,
server_ip String,
server_ip_tags Array(String),
server_port Int32,
server_os_desc String,
server_geolocation LowCardinality(String),
@@ -2783,6 +2814,7 @@ TO tsg_galaxy_v3.monitor_event_local
server_sub_administrative_area String,
server_asn Nullable(Int64),
server_fqdn String,
server_fqdn_tags Array(String),
server_domain String,
app_transition String,
app LowCardinality(String),
@@ -3002,6 +3034,7 @@ SELECT
monitor_mirrored_pkts,
monitor_mirrored_bytes,
client_ip,
client_ip_tags,
client_port,
client_os_desc,
client_geolocation,
@@ -3016,6 +3049,7 @@ SELECT
phone_number,
apn,
server_ip,
server_ip_tags,
server_port,
server_os_desc,
server_geolocation,
@@ -3025,6 +3059,7 @@ SELECT
server_sub_administrative_area,
server_asn,
server_fqdn,
server_fqdn_tags,
server_domain,
app_transition,
app,

8
tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl_check.sql
View File

@@ -2,13 +2,13 @@ SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, asses
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

22
tsg_olap/upgrade/TSG-24.07/hos/hosutil/config.properties Normal file
View File

@@ -0,0 +1,22 @@
qgw.serverAddr=http://192.168.44.67:9999
hos.serverAddr=http://192.168.44.67:9098
hos.token=c21f969b5f03d33d43e04f8f136e7682
kafka.server=192.168.44.11:9092
#延迟时间，校验多少秒之前的文件，单位秒
hos.timeDelay=180
kafka.traffic.topic=TRAFFIC-FILE-STREAM-RECORD
kafka.troubleshooting.topic=TROUBLESHOOTING-FILE-STREAM-RECORD
file.chunk.combiner.window.time=11000
traffic.file.count=0
threads=1
maxThreads=10
printOutInterval=1000
http.maxTotal=100
http.defaultMaxPerRoute=100
http.connectTimeout=5000
http.connectionRequestTimeout=10000
http.socketTimeout=-1
http.staleConnectionCheckEnabled=true
hos.logTypes=security_event,monitor_event,proxy_event,session_record,voip_record,assessment_event,transaction_record,troubleshooting
hos.logTypesFileTypesFileUrlFields=security_event:http-http_response_body&http_request_body,pacp-packet_capture_file,eml-mail_eml_file;proxy_event:http-http_response_body&http_request_body;session_record:http-http_response_body&http_request_body,pacp-packet_capture_file,eml-mail_eml_file;voip_record:pacp-rtp_pcap_path;assessment_event:other-assessment_file;transaction_record:http-http_response_body&http_request_body,eml-mail_eml_file;monitor_event:http-http_response_body&http_request_body,pacp-packet_capture_file,eml-mail_eml_file

BIN
tsg_olap/upgrade/TSG-24.07/hos/hosutil/galaxy-hos-util-1.2.jar → tsg_olap/upgrade/TSG-24.07/hos/hosutil/galaxy-hos-util-1.3.jar
View File

Binary file not shown.

23
tsg_olap/upgrade/TSG-24.07/hos/hosutil/hosutil.sh
View File

@@ -26,12 +26,12 @@ usage() {
echo " -s --startTime 起始时间。时间是UTC时间，格式为yyyyMMdd、yyyy-MM-dd、yyyyMMddHHmmss，默认是前一天的时间。"
echo " -e --endTime 结束时间。时间是UTC时间，格式为yyyyMMdd、yyyy-MM-dd、yyyyMMddHHmmss，默认是当前时间。"
echo " -c --count 评估的日志数量，默认为1000，最大值为100000。"
echo " -l --logType 评估指定日志的文件，不指定该参数则评估所有日志，支持评估多种日志，使用逗号隔开，例如session_record,security_event。支持的日志有security_event、proxy_event、session_record、voip_record、assessment_event、transaction_record、troubleshooting。"
echo " -f --fileType 指定文件的类型，不指定该参数则为所有类型，例如pcap。目前类型有：mail、http、pcap、other。目前只有session_record（mail、http、pcap）、security_event（mail、http、pcap）、transaction_record（mail、http）包含多种类型，其他日志省略该参数即可。"
echo " -d --dataCenter 评估的数据中心，可指定多个，使用逗号隔开，不指定为所有数据中心。只有security_event、monitor_event、proxy_event、session_record、voip_record支持选择数据中心"
echo " -l --logType 评估指定日志的文件，不指定该参数则评估所有日志，支持评估多种日志，使用逗号隔开，例如session_record,security_event。支持的日志有security_event、monitor_event、proxy_event、session_record、voip_record、assessment_event、transaction_record、troubleshooting。"
echo " -f --fileType 指定文件的类型，不指定该参数则为所有类型，例如pcap。目前类型有：mail、http、pcap、other。目前只有session_record（mail、http、pcap）、security_event（mail、http、pcap）、monitor_event（mail、http、pcap）、transaction_record（mail、http）包含多种类型，其他日志省略该参数即可。"
echo " -t --threads 线程数，默认为1，最大值为10。"
echo "combiner options:"
echo " -j --job 要验证的任务，有traffic、troubleshooting、all，all为都验证，默认为all。"
echo " -p --prefix 文件名前缀，防止多次执行文件名重复，不可省略。"
echo " -j --job 要验证的任务，有traffic、troubleshooting，都验证指定为traffic,troubleshooting，不指定该参数默认为traffic。（24.05已删除troubleshooting任务）"
}
if [ $# -eq 0 ]; then
@@ -39,7 +39,7 @@ if [ $# -eq 0 ]; then
exit 0
fi
version="1.2"
version="1.3"
operation=$1
bucket=""
directory=""
@@ -51,7 +51,8 @@ count=1000
threads=1
logType=""
fileType=""
jobName="all"
dataCenter=""
jobName="traffic"
jar="galaxy-hos-util-"$version".jar"
shift
@@ -65,7 +66,11 @@ while getopts ":h:b:d:k:p:s:e:c:t:l:f:j:" opt; do
bucket=$OPTARG
;;
d)
directory=$OPTARG
if [ "$operation" == "check" ]; then
dataCenter=$OPTARG
else
directory=$OPTARG
fi
;;
k)
keys=$OPTARG
@@ -122,11 +127,11 @@ java -jar $jar upload $bucket $directory threadNum=$threads
}
check() {
java -jar $jar check logType=$logType fileType=$fileType maxLogs=$count timeRange=$startTime/$endTime threadNum=$threads
java -jar $jar check dataCenter=$dataCenter logType=$logType fileType=$fileType maxLogs=$count timeRange=$startTime/$endTime threadNum=$threads
}
combiner() {
java -jar $jar combiner $jobName $prefix
java -jar $jar combiner $jobName
}
if [ "$operation" = "download" ];then

3377
tsg_olap/upgrade/TSG-24.08/clickhouse/tsg_olap_clickhouse_ddl_24.08.sql Normal file
View File

File diff suppressed because it is too large Load Diff

22
tsg_olap/upgrade/TSG-24.08/clickhouse/tsg_olap_clickhouse_ddl_check_24.08.sql Normal file
View File

@@ -0,0 +1,22 @@
SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT log_id, recv_time, vsys_id, timestamp_us, egress_action, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
SELECT log_id, recv_time, vsys_id, device_id, device_group, data_center, direction, ip_protocol, client_ip, server_ip, internal_ip, external_ip, client_country, server_country, client_asn, server_asn, server_fqdn, server_domain, app, app_category, c2s_ttl, s2c_ttl, c2s_link_id, s2c_link_id, sessions, bytes, sent_bytes, received_bytes, pkts, sent_pkts, received_pkts, asymmetric_c2s_flows, asymmetric_s2c_flows, c2s_fragments, s2c_fragments, c2s_tcp_lost_bytes, s2c_tcp_lost_bytes, c2s_tcp_retransmitted_pkts, s2c_tcp_retransmitted_pkts
FROM tsg_galaxy_v3.traffic_sketch_metric where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

1034
tsg_olap/upgrade/TSG-24.08/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.08.sql Normal file
View File

File diff suppressed because it is too large Load Diff

143
tsg_olap/upgrade/TSG-24.08/groot_stream/single-cluster-examples/etl_traffic_sketch_metric_kafka_to_clickhouse Normal file
View File

@@ -0,0 +1,143 @@
sources:
kafka_source:
type: kafka
properties:
topic: TRAFFIC-SKETCH-METRIC
kafka.bootstrap.servers: "{{ kafka_source_servers }}"
kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
kafka.group.id: etl_traffic_sketch_metric
kafka.auto.offset.reset: latest
kafka.compression.type: none
format: json
processing_pipelines:
pre_etl_processor: # [object] Processing Pipeline
type: projection
remove_fields:
output_fields:
functions: # [array of object] Function List
- function: UNIX_TIMESTAMP_CONVERTER
lookup_fields: [ timestamp_ms ]
output_fields: [ recv_time ]
parameters:
precision: seconds
interval: 300
aggregate_processor:
type: aggregate
group_by_fields: [vsys_id,device_id,device_group,data_center,ip_protocol,direction,client_ip,server_ip,server_domain,app,recv_time]
window_type: tumbling_processing_time # tumbling_event_time,sliding_processing_time,sliding_event_time
window_size: 300
functions:
- function: NUMBER_SUM
lookup_fields: [ sessions ]
- function: NUMBER_SUM
lookup_fields: [ bytes ]
- function: NUMBER_SUM
lookup_fields: [ sent_bytes ]
- function: NUMBER_SUM
lookup_fields: [ received_bytes ]
- function: NUMBER_SUM
lookup_fields: [ pkts ]
- function: NUMBER_SUM
lookup_fields: [ sent_pkts ]
- function: NUMBER_SUM
lookup_fields: [ received_pkts ]
- function: NUMBER_SUM
lookup_fields: [ asymmetric_c2s_flows ]
- function: NUMBER_SUM
lookup_fields: [ asymmetric_s2c_flows ]
- function: NUMBER_SUM
lookup_fields: [ c2s_fragments ]
- function: NUMBER_SUM
lookup_fields: [ s2c_fragments ]
- function: NUMBER_SUM
lookup_fields: [ c2s_tcp_lost_bytes ]
- function: NUMBER_SUM
lookup_fields: [ s2c_tcp_lost_bytes ]
- function: NUMBER_SUM
lookup_fields: [ c2s_tcp_retransmitted_pkts ]
- function: NUMBER_SUM
lookup_fields: [ s2c_tcp_retransmitted_pkts ]
- function: FIRST_VALUE
lookup_fields: [ client_country ]
- function: FIRST_VALUE
lookup_fields: [ server_country ]
- function: FIRST_VALUE
lookup_fields: [ client_asn ]
- function: FIRST_VALUE
lookup_fields: [ server_asn ]
- function: FIRST_VALUE
lookup_fields: [ server_fqdn ]
- function: FIRST_VALUE
lookup_fields: [ app_category ]
- function: FIRST_VALUE
lookup_fields: [ c2s_ttl ]
- function: FIRST_VALUE
lookup_fields: [ s2c_ttl ]
- function: FIRST_VALUE
lookup_fields: [ c2s_link_id ]
- function: FIRST_VALUE
lookup_fields: [ s2c_link_id ]
post_etl_processor: # [object] Processing Pipeline
type: projection
remove_fields:
output_fields:
functions: # [array of object] Function List
- function: EVAL
output_fields: [ internal_ip ]
parameters:
value_expression: 'direction=Outbound? client_ip : server_ip'
- function: EVAL
output_fields: [ external_ip ]
parameters:
value_expression: 'direction=Outbound? server_ip : client_ip'
- function: SNOWFLAKE_ID
lookup_fields: [ '' ]
output_fields: [ log_id ]
filter:
parameters:
data_center_id_num: 1
sinks:
clickhouse_sink:
type: clickhouse
properties:
host: "{{ clickhouse_servers }}"
table: tsg_galaxy_v3.traffic_sketch_metric_local
batch.size: 100000
batch.interval: 30s
connection.user: e54c9568586180eede1506eecf3574e9
connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
application:
env: # [object] Environment Variables
name: etl_traffic_sketch_metric # [string] Job Name
shade.identifier: aes
pipeline:
object-reuse: true # [boolean] Object Reuse, default is false
topology:
topology:
- name: kafka_source
downstream: [pre_etl_processor]
- name: pre_etl_processor
downstream: [aggregate_processor]
- name: aggregate_processor
downstream: [post_etl_processor]
- name: post_etl_processor
downstream: [clickhouse_sink]
- name: clickhouse_sink

152
tsg_olap/upgrade/TSG-24.08/groot_stream/templates/traffic_sketch_metric.yaml.j2 Normal file
View File

@@ -0,0 +1,152 @@
sources:
kafka_source:
type: kafka
properties:
topic: TRAFFIC-SKETCH-METRIC
kafka.bootstrap.servers: {{ kafka_source_servers }}
kafka.session.timeout.ms: 60000
kafka.max.poll.records: 3000
kafka.max.partition.fetch.bytes: 31457280
kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
kafka.group.id: etl_traffic_sketch_metric
kafka.auto.offset.reset: latest
kafka.compression.type: none
format: json
processing_pipelines:
pre_etl_processor: # [object] Processing Pipeline
type: projection
remove_fields:
output_fields:
functions: # [array of object] Function List
- function: UNIX_TIMESTAMP_CONVERTER
lookup_fields: [ timestamp_ms ]
output_fields: [ recv_time ]
parameters:
precision: seconds
interval: 300
aggregate_processor:
type: aggregate
group_by_fields: [vsys_id,device_id,device_group,data_center,ip_protocol,direction,client_ip,server_ip,server_domain,app,recv_time]
window_type: tumbling_processing_time # tumbling_event_time,sliding_processing_time,sliding_event_time
window_size: 300
functions:
- function: NUMBER_SUM
lookup_fields: [ sessions ]
- function: NUMBER_SUM
lookup_fields: [ bytes ]
- function: NUMBER_SUM
lookup_fields: [ sent_bytes ]
- function: NUMBER_SUM
lookup_fields: [ received_bytes ]
- function: NUMBER_SUM
lookup_fields: [ pkts ]
- function: NUMBER_SUM
lookup_fields: [ sent_pkts ]
- function: NUMBER_SUM
lookup_fields: [ received_pkts ]
- function: NUMBER_SUM
lookup_fields: [ asymmetric_c2s_flows ]
- function: NUMBER_SUM
lookup_fields: [ asymmetric_s2c_flows ]
- function: NUMBER_SUM
lookup_fields: [ c2s_fragments ]
- function: NUMBER_SUM
lookup_fields: [ s2c_fragments ]
- function: NUMBER_SUM
lookup_fields: [ c2s_tcp_lost_bytes ]
- function: NUMBER_SUM
lookup_fields: [ s2c_tcp_lost_bytes ]
- function: NUMBER_SUM
lookup_fields: [ c2s_tcp_retransmitted_pkts ]
- function: NUMBER_SUM
lookup_fields: [ s2c_tcp_retransmitted_pkts ]
- function: FIRST_VALUE
lookup_fields: [ client_country ]
- function: FIRST_VALUE
lookup_fields: [ server_country ]
- function: FIRST_VALUE
lookup_fields: [ client_asn ]
- function: FIRST_VALUE
lookup_fields: [ server_asn ]
- function: FIRST_VALUE
lookup_fields: [ server_fqdn ]
- function: FIRST_VALUE
lookup_fields: [ app_category ]
- function: FIRST_VALUE
lookup_fields: [ c2s_ttl ]
- function: FIRST_VALUE
lookup_fields: [ s2c_ttl ]
- function: FIRST_VALUE
lookup_fields: [ c2s_link_id ]
- function: FIRST_VALUE
lookup_fields: [ s2c_link_id ]
post_etl_processor: # [object] Processing Pipeline
type: projection
remove_fields:
output_fields:
functions: # [array of object] Function List
- function: EVAL
output_fields: [ internal_ip ]
parameters:
value_expression: 'direction=Outbound? client_ip : server_ip'
- function: EVAL
output_fields: [ external_ip ]
parameters:
value_expression: 'direction=Outbound? server_ip : client_ip'
- function: SNOWFLAKE_ID
lookup_fields: [ '' ]
output_fields: [ log_id ]
filter:
parameters:
data_center_id_num: 1
sinks:
kafka_sink:
type: kafka
properties:
topic: TRAFFIC-SKETCH-METRIC
kafka.bootstrap.servers: {{ kafka_sink_servers }}
kafka.retries: 0
kafka.linger.ms: 10
kafka.request.timeout.ms: 30000
kafka.batch.size: 262144
kafka.buffer.memory: 134217728
kafka.max.request.size: 10485760
kafka.compression.type: snappy
kafka.security.protocol: SASL_PLAINTEXT
kafka.sasl.mechanism: PLAIN
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
format: json
json.ignore.parse.errors: false
log.failures.only: true
clickhouse_sink:
type: clickhouse
properties:
host: {{ clickhouse_servers }}
table: tsg_galaxy_v3.traffic_sketch_metric_local
batch.size: 100000
batch.interval: 30s
connection.user: e54c9568586180eede1506eecf3574e9
connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
application:
env: # [object] Environment Variables
name: etl_traffic_sketch_metric # [string] Job Name
shade.identifier: aes
pipeline:
object-reuse: true # [boolean] Object Reuse, default is false
{{ topology }}

24
tsg_olap/upgrade/TSG-24.08/groot_stream/udf.plugins Normal file
View File

@@ -0,0 +1,24 @@
com.geedgenetworks.core.udf.AsnLookup
com.geedgenetworks.core.udf.CurrentUnixTimestamp
com.geedgenetworks.core.udf.DecodeBase64
com.geedgenetworks.core.udf.Domain
com.geedgenetworks.core.udf.Drop
com.geedgenetworks.core.udf.EncodeBase64
com.geedgenetworks.core.udf.Eval
com.geedgenetworks.core.udf.Flatten
com.geedgenetworks.core.udf.FromUnixTimestamp
com.geedgenetworks.core.udf.GenerateStringArray
com.geedgenetworks.core.udf.GeoIpLookup
com.geedgenetworks.core.udf.JsonExtract
com.geedgenetworks.core.udf.PathCombine
com.geedgenetworks.core.udf.Rename
com.geedgenetworks.core.udf.SnowflakeId
com.geedgenetworks.core.udf.StringJoiner
com.geedgenetworks.core.udf.UnixTimestampConverter
com.geedgenetworks.core.udf.udaf.NumberSum
com.geedgenetworks.core.udf.udaf.CollectList
com.geedgenetworks.core.udf.udaf.CollectSet
com.geedgenetworks.core.udf.udaf.LongCount
com.geedgenetworks.core.udf.udaf.Mean
com.geedgenetworks.core.udf.udaf.LastValue
com.geedgenetworks.core.udf.udaf.FirstValue