Merge branch 'master' of git.mesalab.cn:galaxy/deployment/updata-record

cyber_narrator/installation/clickhouse/cn_clickhouse_ddl.sql

@@ -884,6 +884,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,
@@ -899,6 +901,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,
@@ -1486,3 +1490,221 @@ GROUP BY
     ip,
     l7_protocol,
     port;
+
+
+CREATE TABLE cyber_narrator_galaxy.match_indicator_local ON CLUSTER ck_cluster (
+    indicator_fields String,
+    indicator_values String,
+    match_num Int64,
+    reset Int64,
+    client_ip String,
+    client_country_region String,
+    client_super_admin_area String,
+    client_admin_area String,
+    client_longitude Nullable(Float64),
+    client_latitude Nullable(Float64),
+    server_ip String,
+    server_country_region String,
+    server_super_admin_area String,
+    server_admin_area String,
+    server_longitude Nullable(Float64),
+    server_latitude Nullable(Float64),
+    domain String,
+    app String,
+    match_time Int64,
+    match_id UInt64,
+    rule_id UInt64,
+    rule_version String,
+    rule_type String,
+    is_builtin Int64,
+    event_type String,
+    event_name String,
+    severity Int64
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(match_time))
+ORDER BY (match_id, match_time);
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_cluster (
+    indicator_fields String,
+    indicator_values String,
+    match_num Int64,
+    reset Int64,
+    client_ip String,
+    client_country_region String,
+    client_super_admin_area String,
+    client_admin_area String,
+    client_longitude Nullable(Float64),
+    client_latitude Nullable(Float64),
+    server_ip String,
+    server_country_region String,
+    server_super_admin_area String,
+    server_admin_area String,
+    server_longitude Nullable(Float64),
+    server_latitude Nullable(Float64),
+    domain String,
+    app String,
+    match_time Int64,
+    match_id UInt64,
+    rule_id UInt64,
+    rule_version String,
+    rule_type String,
+    is_builtin Int64,
+    event_type String,
+    event_name String,
+    severity Int64
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_indicator_local', rand());
+
+
+CREATE TABLE cyber_narrator_galaxy.match_threshold_local ON CLUSTER ck_cluster (
+    key_fields String,
+    key_values String,
+    threshold_value Float32,
+    metric_value Float32,
+    unit Int64 DEFAULT 1,
+	reset Int64,
+    start_time Int64,
+    end_time Int64,
+    match_id UInt64,
+    rule_id UInt64,
+    rule_version String,
+    rule_type String,
+    is_builtin Int64,
+    event_type String,
+    event_name String,
+    severity Int64
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(start_time))
+ORDER BY (match_id, start_time);
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_cluster (
+    key_fields String,
+    key_values String,
+    threshold_value Float32,
+    metric_value Float32,
+	unit Int64 DEFAULT 1,
+    reset Int64,
+    start_time Int64,
+    end_time Int64,
+    match_id UInt64,
+    rule_id UInt64,
+    rule_version String,
+    rule_type String,
+    is_builtin Int64,
+    event_type String,
+    event_name String,
+    severity Int64
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_threshold_local', rand());
+
+
+CREATE TABLE cyber_narrator_galaxy.match_sequence_local ON CLUSTER ck_cluster (
+    key_fields String,
+    key_values String,
+    event_info String,
+    start_time Int64,
+    end_time Int64,
+    match_id UInt64,
+    rule_id UInt64,
+    rule_version String,
+    rule_type String,
+    is_builtin Int64,
+    event_type String,
+    event_name String,
+    severity Int64
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(start_time))
+ORDER BY (match_id, start_time);
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_sequence ON CLUSTER ck_cluster (
+    key_fields String,
+    key_values String,
+    event_info String,
+    start_time Int64,
+    end_time Int64,
+    match_id UInt64,
+    rule_id UInt64,
+    rule_version String,
+    rule_type String,
+    is_builtin Int64,
+    event_type String,
+    event_name String,
+    severity Int64
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_sequence_local', rand());
+
+
+CREATE TABLE cyber_narrator_galaxy.match_unordered_sequence_local ON CLUSTER ck_cluster (
+    key_fields String,
+    key_values String,
+    event_info String,
+    start_time Int64,
+    end_time Int64,
+    match_id UInt64,
+    rule_id UInt64,
+    rule_version String,
+    rule_type String,
+    is_builtin Int64,
+    event_type String,
+    event_name String,
+    severity Int64
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(start_time))
+ORDER BY (match_id, start_time);
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_unordered_sequence ON CLUSTER ck_cluster (
+    key_fields String,
+    key_values String,
+    event_info String,
+    start_time Int64,
+    end_time Int64,
+    match_id UInt64,
+    rule_id UInt64,
+    rule_version String,
+    rule_type String,
+    is_builtin Int64,
+    event_type String,
+    event_name String,
+    severity Int64
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'match_unordered_sequence_local', rand());
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event_local ON CLUSTER ck_cluster(
+    event_id UInt64, 
+    match_ids SimpleAggregateFunction(anyLast, String),
+    key_fields SimpleAggregateFunction(anyLast, String),
+    key_values SimpleAggregateFunction(anyLast, String),
+    rule_id SimpleAggregateFunction(anyLast, Int64),
+    rule_version SimpleAggregateFunction(anyLast, String),
+    rule_type SimpleAggregateFunction(anyLast, Int8),
+    is_builtin SimpleAggregateFunction(anyLast, Int8),
+    event_type SimpleAggregateFunction(anyLast, String),
+    event_name SimpleAggregateFunction(anyLast, String),
+    reset SimpleAggregateFunction(anyLast, Int64),
+    start_time SimpleAggregateFunction(min, Int64),
+    end_time SimpleAggregateFunction(max, Int64),
+    duration_s SimpleAggregateFunction(max, Int64),
+    status SimpleAggregateFunction(max, Int8)
+) 
+ENGINE=AggregatingMergeTree ORDER BY event_id;
+
+CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event ON CLUSTER ck_cluster(
+    event_id UInt64, 
+    match_ids SimpleAggregateFunction(anyLast, String),
+    key_fields SimpleAggregateFunction(anyLast, String),
+    key_values SimpleAggregateFunction(anyLast, String),
+    rule_id SimpleAggregateFunction(anyLast, Int64),
+    rule_version SimpleAggregateFunction(anyLast, String),
+    rule_type SimpleAggregateFunction(anyLast, Int8),
+    is_builtin SimpleAggregateFunction(anyLast, Int8),
+    event_type SimpleAggregateFunction(anyLast, String),
+    event_name SimpleAggregateFunction(anyLast, String),
+    reset SimpleAggregateFunction(anyLast, Int64),
+    start_time SimpleAggregateFunction(min, Int64),
+    end_time SimpleAggregateFunction(max, Int64),
+    duration_s SimpleAggregateFunction(max, Int64),
+    status SimpleAggregateFunction(max, Int8)
+)
+ENGINE = Distributed('ck_cluster',
+ 'cyber_narrator_galaxy',
+ 'cn_event_local',
+ rand());

cyber_narrator/installation/clickhouse/cn_clickhouse_ddl_check.sql

@@ -51,7 +51,18 @@ SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts,
 FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state 
 FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
+SELECT subscriber_id, imei, imsi, phone_number, apn, cell_id, cell_type, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
 FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
 FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+
+SELECT indicator_fields, indicator_values, match_num, reset, client_ip, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, server_ip, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, domain, app, match_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
+FROM cyber_narrator_galaxy.match_indicator where match_time >= toUnixTimestamp('2030-01-01 00:00:00') AND match_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT key_fields, key_values, threshold_value, metric_value, unit, reset, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
+FROM cyber_narrator_galaxy.match_threshold where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
+FROM cyber_narrator_galaxy.match_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
+FROM cyber_narrator_galaxy.match_unordered_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT event_id, match_ids, key_fields, key_values, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, reset,start_time, end_time, duration_s, status
+FROM cyber_narrator_galaxy.cn_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');

cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl.sql

@@ -1306,6 +1306,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,
@@ -1321,6 +1323,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,
@@ -1336,6 +1340,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,
@@ -2225,9 +2231,10 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_q
 CREATE TABLE cyber_narrator_galaxy.match_threshold_local ON CLUSTER ck_cluster (
     key_fields String,
     key_values String,
-    threshold_num Int64,
+    threshold_value Float32,
-    records_num Int64,
+    metric_value Float32,
-    reset Int64,
+    unit Int64 DEFAULT 1,
+	reset Int64,
     start_time Int64,
     end_time Int64,
     match_id UInt64,
@@ -2246,9 +2253,10 @@ ORDER BY (match_id, start_time);
 CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_cluster (
     key_fields String,
     key_values String,
-    threshold_num Int64,
+    threshold_value Float32,
-    records_num Int64,
+    metric_value Float32,
-    reset Int64,
+    unit Int64 DEFAULT 1,
+	reset Int64,
     start_time Int64,
     end_time Int64,
     match_id UInt64,
@@ -2264,9 +2272,10 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_c
 CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_query (
     key_fields String,
     key_values String,
-    threshold_num Int64,
+    threshold_value Float32,
-    records_num Int64,
+    metric_value Float32,
-    reset Int64,
+    unit Int64 DEFAULT 1,
+	reset Int64,
     start_time Int64,
     end_time Int64,
     match_id UInt64,

cyber_narrator/installation/clickhouse/旧部署模式建表语句/cn_clickhouse_ddl_check.sql

@@ -51,16 +51,18 @@ SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts,
 FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state 
 FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
+SELECT subscriber_id, imei, imsi, phone_number, apn, cell_id, cell_type, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
 FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
 FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 
 SELECT indicator_fields, indicator_values, match_num, reset, client_ip, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, server_ip, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, domain, app, match_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
 FROM cyber_narrator_galaxy.match_indicator where match_time >= toUnixTimestamp('2030-01-01 00:00:00') AND match_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT key_fields, key_values, threshold_num, records_num, reset, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
+SELECT key_fields, key_values, threshold_value, metric_value, unit, reset, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
 FROM cyber_narrator_galaxy.match_threshold where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
 FROM cyber_narrator_galaxy.match_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
 FROM cyber_narrator_galaxy.match_unordered_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT event_id, match_ids, key_fields, key_values, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, reset,start_time, end_time, duration_s, status
+FROM cyber_narrator_galaxy.cn_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');

cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_24.04.sql

@@ -884,6 +884,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,
@@ -899,6 +901,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,

cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_check_24.04.sql

@@ -51,7 +51,7 @@ SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts,
 FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state 
 FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
+SELECT subscriber_id, imei, imsi, phone_number, apn, cell_id, cell_type, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
 FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
 FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

cyber_narrator/upgrade/2024/CN-24.04/clickhouse/cn_clickhouse_ddl_upgrade_24.04.sql

@@ -245,4 +245,11 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.metric_tag ON CLUSTER ck_cluste
     domain_sketch String,
     ip_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(ip_sketch),
     domain_sketch_agg_state AggregateFunction(uniqTheta,String) MATERIALIZED base64Decode(domain_sketch)
-) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
+) ENGINE = Distributed('ck_cluster', 'cyber_narrator_galaxy', 'metric_tag_local', rand());
+
+
+ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_type Int64 after apn;
+ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_type Int64 after apn;
+
+ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;
+ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;

cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_24.08.sql

@@ -884,6 +884,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber_local ON CL
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,
@@ -899,6 +901,8 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.location_subscriber ON CLUSTER
  imsi String,
  phone_number String,
  apn String,
+ cell_id String,
+ cell_type Int64,
  subscriber_longitude Nullable(Float64),
  subscriber_latitude Nullable(Float64),
  first_location String,
@@ -1555,9 +1559,10 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_c
 CREATE TABLE cyber_narrator_galaxy.match_threshold_local ON CLUSTER ck_cluster (
     key_fields String,
     key_values String,
-    threshold_num Int64,
+    threshold_value Float32,
-    records_num Int64,
+    metric_value Float32,
-    reset Int64,
+    unit Int64 DEFAULT 1,
+	reset Int64,
     start_time Int64,
     end_time Int64,
     match_id UInt64,
@@ -1576,8 +1581,9 @@ ORDER BY (match_id, start_time);
 CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_cluster (
     key_fields String,
     key_values String,
-    threshold_num Int64,
+    threshold_value Float32,
-    records_num Int64,
+    metric_value Float32,
+	unit Int64 DEFAULT 1,
     reset Int64,
     start_time Int64,
     end_time Int64,
@@ -1673,6 +1679,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event_local ON CLUSTER ck_cl
     is_builtin SimpleAggregateFunction(anyLast, Int8),
     event_type SimpleAggregateFunction(anyLast, String),
     event_name SimpleAggregateFunction(anyLast, String),
+    reset SimpleAggregateFunction(anyLast, Int64),
     start_time SimpleAggregateFunction(min, Int64),
     end_time SimpleAggregateFunction(max, Int64),
     duration_s SimpleAggregateFunction(max, Int64),
@@ -1691,6 +1698,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event ON CLUSTER ck_cluster(
     is_builtin SimpleAggregateFunction(anyLast, Int8),
     event_type SimpleAggregateFunction(anyLast, String),
     event_name SimpleAggregateFunction(anyLast, String),
+    reset SimpleAggregateFunction(anyLast, Int64),
     start_time SimpleAggregateFunction(min, Int64),
     end_time SimpleAggregateFunction(max, Int64),
     duration_s SimpleAggregateFunction(max, Int64),

cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_check_24.08.sql

@@ -51,18 +51,18 @@ SELECT subscriber_id, app, imei, imsi, phone_number, apn, stat_time, sent_pkts,
 FROM cyber_narrator_galaxy.metric_subscriber_app where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT tag, stat_time, ip_sketch, domain_sketch,ip_sketch_agg_state, domain_sketch_agg_state 
 FROM cyber_narrator_galaxy.metric_tag where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT subscriber_id, imei, imsi, phone_number, apn, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
+SELECT subscriber_id, imei, imsi, phone_number, apn, cell_id, cell_type, subscriber_longitude, subscriber_latitude, first_location, second_location, third_location,data_source,  stat_time
 FROM cyber_narrator_galaxy.location_subscriber where stat_time >= toUnixTimestamp('2030-01-01 00:00:00') AND stat_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, flags, start_timestamp_ms, end_timestamp_ms, duration_ms, decoded_as, client_ip, server_ip, client_port, server_port, app, app_transition, decoded_path, ip_protocol, l7_protocol, out_link_id, in_link_id, subscriber_id, imei, imsi, phone_number, apn, http_host, http_url, http_cookie, http_referer, http_user_agent, http_request_line, http_response_line, http_status_code, ssl_version, ssl_sni, ssl_san, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, dns_qr, dns_opcode, dns_aa, dns_rcode, dns_qname, dns_qtype, dns_qclass, dns_sub, dns_rr, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, out_link_direction, in_link_direction, `domain`, domain_sld, domain_category_name, domain_category_group, domain_reputation_level, domain_icp_company_name, domain_whois_org, domain_tags, client_zone, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, client_isp, client_asn, client_ip_tags, server_zone, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, server_isp, server_asn, server_ip_tags, app_category, app_subcategory, app_company, app_company_category, app_tags, sent_pkts, sent_bytes, received_pkts, received_bytes, sessions, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_rtt_ms, http_response_latency_ms, ssl_handshake_latency_ms, dns_response_latency_ms
 FROM cyber_narrator_galaxy.session_record_cn where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 
 SELECT indicator_fields, indicator_values, match_num, reset, client_ip, client_country_region, client_super_admin_area, client_admin_area, client_longitude, client_latitude, server_ip, server_country_region, server_super_admin_area, server_admin_area, server_longitude, server_latitude, domain, app, match_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
 FROM cyber_narrator_galaxy.match_indicator where match_time >= toUnixTimestamp('2030-01-01 00:00:00') AND match_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT key_fields, key_values, threshold_num, records_num, reset, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
+SELECT key_fields, key_values, threshold_value, metric_value, unit, reset, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
 FROM cyber_narrator_galaxy.match_threshold where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity 
 FROM cyber_narrator_galaxy.match_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT key_fields, key_values, event_info, start_time, end_time, match_id, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, severity
 FROM cyber_narrator_galaxy.match_unordered_sequence where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT event_id, match_ids, key_fields, key_values, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, start_time, end_time, duration_s, status
+SELECT event_id, match_ids, key_fields, key_values, rule_id, rule_version, rule_type, is_builtin, event_type, event_name, reset,start_time, end_time, duration_s, status
 FROM cyber_narrator_galaxy.cn_event where start_time >= toUnixTimestamp('2030-01-01 00:00:00') AND start_time <toUnixTimestamp('2030-01-01 00:00:01');

cyber_narrator/upgrade/2024/CN-24.08/clickhouse/cn_clickhouse_ddl_upgrade_24.08.sql

@@ -1,3 +1,5 @@
+set distributed_ddl_task_timeout = 180;
+
 CREATE TABLE cyber_narrator_galaxy.match_indicator_local ON CLUSTER ck_cluster (
     indicator_fields String,
     indicator_values String,
@@ -65,9 +67,10 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_indicator ON CLUSTER ck_c
 CREATE TABLE cyber_narrator_galaxy.match_threshold_local ON CLUSTER ck_cluster (
     key_fields String,
     key_values String,
-    threshold_num Int64,
+    threshold_value Float32,
-    records_num Int64,
+    metric_value Float32,
-    reset Int64,
+    unit Int64 DEFAULT 1,
+	reset Int64,
     start_time Int64,
     end_time Int64,
     match_id UInt64,
@@ -86,8 +89,9 @@ ORDER BY (match_id, start_time);
 CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.match_threshold ON CLUSTER ck_cluster (
     key_fields String,
     key_values String,
-    threshold_num Int64,
+    threshold_value Float32,
-    records_num Int64,
+    metric_value Float32,
+	unit Int64 DEFAULT 1,
     reset Int64,
     start_time Int64,
     end_time Int64,
@@ -184,6 +188,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event_local ON CLUSTER ck_cl
     is_builtin SimpleAggregateFunction(anyLast, Int8),
     event_type SimpleAggregateFunction(anyLast, String),
     event_name SimpleAggregateFunction(anyLast, String),
+    reset SimpleAggregateFunction(anyLast, Int64),
     start_time SimpleAggregateFunction(min, Int64),
     end_time SimpleAggregateFunction(max, Int64),
     duration_s SimpleAggregateFunction(max, Int64),
@@ -202,6 +207,7 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event ON CLUSTER ck_cluster(
     is_builtin SimpleAggregateFunction(anyLast, Int8),
     event_type SimpleAggregateFunction(anyLast, String),
     event_name SimpleAggregateFunction(anyLast, String),
+    reset SimpleAggregateFunction(anyLast, Int64),
     start_time SimpleAggregateFunction(min, Int64),
     end_time SimpleAggregateFunction(max, Int64),
     duration_s SimpleAggregateFunction(max, Int64),
@@ -210,4 +216,11 @@ CREATE TABLE IF NOT EXISTS cyber_narrator_galaxy.cn_event ON CLUSTER ck_cluster(
 ENGINE = Distributed('ck_cluster',
  'cyber_narrator_galaxy',
  'cn_event_local',
- rand());
+ rand());
+ 
+ 
+ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_type Int64 after apn;
+ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_type Int64 after apn;
+
+ALTER table cyber_narrator_galaxy.location_subscriber_local ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;
+ALTER table cyber_narrator_galaxy.location_subscriber ON CLUSTER ck_cluster add column IF NOT EXISTS cell_id String after apn;

tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl.sql

@@ -296,6 +296,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -545,6 +546,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -792,6 +794,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1040,6 +1043,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1287,6 +1291,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1535,6 +1540,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1661,7 +1667,8 @@ sip_responder_sdp_media_port Nullable(Int32),
 sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
-sip_bye String
+sip_bye String,
+sip_bye_reason String
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(toDate(recv_time))
@@ -1755,7 +1762,8 @@ sip_responder_sdp_media_port Nullable(Int32),
 sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
-sip_bye String
+sip_bye String,
+sip_bye_reason String
 )
 ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand());
 
@@ -1824,6 +1832,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1896,6 +1905,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -2453,6 +2463,7 @@ TO tsg_galaxy_v3.security_event_local
     sip_responder_sdp_content String,
     sip_duration_s Nullable(Int32),
     sip_bye String,
+    sip_bye_reason String,
     rtp_payload_type_c2s Nullable(Int32),
     rtp_payload_type_s2c Nullable(Int32),
     rtp_pcap_path String,
@@ -2698,6 +2709,7 @@ SELECT
     sip_responder_sdp_content,
     sip_duration_s,
     sip_bye,
+    sip_bye_reason,
     rtp_payload_type_c2s,
     rtp_payload_type_s2c,
     rtp_pcap_path,
@@ -2948,6 +2960,7 @@ TO tsg_galaxy_v3.monitor_event_local
     sip_responder_sdp_content String,
     sip_duration_s Nullable(Int32),
     sip_bye String,
+    sip_bye_reason String,
     rtp_payload_type_c2s Nullable(Int32),
     rtp_payload_type_s2c Nullable(Int32),
     rtp_pcap_path String,
@@ -3193,6 +3206,7 @@ SELECT
     sip_responder_sdp_content,
     sip_duration_s,
     sip_bye,
+    sip_bye_reason,
     rtp_payload_type_c2s,
     rtp_payload_type_s2c,
     rtp_pcap_path,

tsg_olap/installation/clickhouse/最新全量建表语句/tsg_olap_clickhouse_ddl_check.sql

@@ -2,17 +2,17 @@ SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, asses
 FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
 FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
+SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason
 FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
 FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT log_id, recv_time, vsys_id, timestamp_us, egress_action, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
 FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

tsg_olap/installation/flink/groot_stream/templates/data_transporter.yaml.j2

@@ -50,7 +50,6 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
 
 

tsg_olap/installation/flink/groot_stream/templates/datapath_telemetry_record.yaml.j2

@@ -73,7 +73,5 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
 
-

tsg_olap/installation/flink/groot_stream/templates/dos_event_kafka_to_clickhouse

@@ -42,9 +42,11 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true # [boolean] Object Reuse, default is false
-  topology:
+  {{ topology }}
-  - name: kafka_source
+
-    downstream: [clickhouse_sink]
+#  topology:
-  - name: clickhouse_sink
+#  - name: kafka_source
+#    downstream: [clickhouse_sink]
+#  - name: clickhouse_sink
 
 

tsg_olap/installation/flink/groot_stream/templates/proxy_event.yaml.j2

@@ -143,6 +143,5 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  topology:
   {{ topology }}
 

tsg_olap/installation/flink/groot_stream/templates/session_record.yaml.j2

@@ -143,4 +143,11 @@ application:
     shade.identifier: aes
     pipeline:
       object-reuse: true
-  {{ topology }}
+  {{ topology }}
+
+#  topology:
+#   - name: kafka_source
+#     downstream: [etl_processor]
+#   - name: etl_processor
+#     downstream: [clickhouse_sink]
+#   - name: clickhouse_sink

tsg_olap/installation/flink/groot_stream/templates/traffic_sketch_metric.yaml.j2

@@ -4,13 +4,12 @@ sources:
     properties:
       topic: TRAFFIC-SKETCH-METRIC
       kafka.bootstrap.servers: {{ kafka_source_servers }}
-      kafka.client.id: TRAFFIC-SKETCH-METRIC
       kafka.session.timeout.ms: 60000
       kafka.max.poll.records: 3000
       kafka.max.partition.fetch.bytes: 31457280
       kafka.security.protocol: SASL_PLAINTEXT
       kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
       kafka.group.id: etl_traffic_sketch_metric
       kafka.auto.offset.reset: latest
       kafka.compression.type: none
@@ -18,29 +17,16 @@ sources:
 
 processing_pipelines:
   etl_processor: # [object] Processing Pipeline
-    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    type: projection
     remove_fields:
     output_fields:
     functions: # [array of object] Function List
-
+      - function: UNIX_TIMESTAMP_CONVERTER
-      - function: FLATTEN
+        lookup_fields: [ timestamp_ms ]
-        lookup_fields: [ fields,tags ]
+        output_fields: [ recv_time ]
-        output_fields: [  ]
         parameters:
-          #prefix: ""
+          precision: seconds
-          depth:  3
+          interval: 60
-         # delimiter: "."
-
-      - function: RENAME
-        lookup_fields: [ '' ]
-        output_fields: [ '' ]
-        filter:
-        parameters:
-         # parent_fields: [tags]
-          #rename_fields:
-            #  tags: tags
-          rename_expression: key =string.replace_all(key,'tags.','');key =string.replace_all(key,'fields.','');return key;
-
       - function: EVAL
         output_fields: [ internal_ip ]
         parameters:
@@ -49,13 +35,6 @@ processing_pipelines:
         output_fields: [ external_ip ]
         parameters:
           value_expression:  'direction=Outbound? server_ip : client_ip'
-
-      - function: UNIX_TIMESTAMP_CONVERTER
-        lookup_fields: [ timestamp_ms ]
-        output_fields: [ recv_time ]
-        parameters:
-          precision: seconds
-
       - function: SNOWFLAKE_ID
         lookup_fields: [ '' ]
         output_fields: [ log_id ]
@@ -70,7 +49,6 @@ sinks:
     properties:
       topic: TRAFFIC-SKETCH-METRIC
       kafka.bootstrap.servers: {{ kafka_sink_servers }}
-      kafka.client.id: TRAFFIC-SKETCH-METRIC
       kafka.retries: 0
       kafka.linger.ms: 10
       kafka.request.timeout.ms: 30000

tsg_olap/upgrade/TSG-24.02.1/clickhouse/tsg_olap_clickhouse_ddl_check_24.02.1.sql

@@ -0,0 +1,20 @@
+SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
+FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT vsys_id, recv_time, log_id, profile_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, session_rate, packet_rate, bit_rate
+FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason
+FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_asn, server_fqdn, server_domain, app_transition, app, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
+
+
+
+    

tsg_olap/upgrade/TSG-24.08/clickhouse/tsg_olap_clickhouse_ddl_24.08.sql

@@ -296,6 +296,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -545,6 +546,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -792,6 +794,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1040,6 +1043,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1287,6 +1291,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1535,6 +1540,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1661,7 +1667,8 @@ sip_responder_sdp_media_port Nullable(Int32),
 sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
-sip_bye String
+sip_bye String,
+sip_bye_reason String
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(toDate(recv_time))
@@ -1755,7 +1762,8 @@ sip_responder_sdp_media_port Nullable(Int32),
 sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
-sip_bye String
+sip_bye String,
+sip_bye_reason String
 )
 ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand());
 
@@ -1824,6 +1832,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -1896,6 +1905,7 @@ sip_responder_sdp_media_type String,
 sip_responder_sdp_content String,
 sip_duration_s Nullable(Int32),
 sip_bye String,
+sip_bye_reason String,
 rtp_payload_type_c2s Nullable(Int32),
 rtp_payload_type_s2c Nullable(Int32),
 rtp_pcap_path String,
@@ -2453,6 +2463,7 @@ TO tsg_galaxy_v3.security_event_local
     sip_responder_sdp_content String,
     sip_duration_s Nullable(Int32),
     sip_bye String,
+    sip_bye_reason String,
     rtp_payload_type_c2s Nullable(Int32),
     rtp_payload_type_s2c Nullable(Int32),
     rtp_pcap_path String,
@@ -2698,6 +2709,7 @@ SELECT
     sip_responder_sdp_content,
     sip_duration_s,
     sip_bye,
+    sip_bye_reason,
     rtp_payload_type_c2s,
     rtp_payload_type_s2c,
     rtp_pcap_path,
@@ -2948,6 +2960,7 @@ TO tsg_galaxy_v3.monitor_event_local
     sip_responder_sdp_content String,
     sip_duration_s Nullable(Int32),
     sip_bye String,
+    sip_bye_reason String,
     rtp_payload_type_c2s Nullable(Int32),
     rtp_payload_type_s2c Nullable(Int32),
     rtp_pcap_path String,
@@ -3193,6 +3206,7 @@ SELECT
     sip_responder_sdp_content,
     sip_duration_s,
     sip_bye,
+    sip_bye_reason,
     rtp_payload_type_c2s,
     rtp_payload_type_s2c,
     rtp_pcap_path,

tsg_olap/upgrade/TSG-24.08/clickhouse/tsg_olap_clickhouse_ddl_check_24.08.sql

@@ -2,17 +2,17 @@ SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, asses
 FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
 FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
 FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
+SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason
 FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
-SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
+SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, sip_bye_reason, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
 FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
 SELECT log_id, recv_time, vsys_id, timestamp_us, egress_action, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
 FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');

tsg_olap/upgrade/TSG-24.08/clickhouse/tsg_olap_clickhouse_ddl_upgrade_24.08.sql

@@ -1,5 +1,3 @@
-set distributed_ddl_task_timeout = 180;
-
 drop view if exists tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster;
 drop view if exists tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster;
 
@@ -42,6 +40,23 @@ ALTER table tsg_galaxy_v3.monitor_event on cluster ck_cluster add column IF NOT
 ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS server_fqdn_tags Array(String) after server_fqdn;
 ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS server_fqdn_tags Array(String) after server_fqdn;
 
+-- TSG-22310 clickhouse相关表SIP协议新增sip_bye_reason字段
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+
+ALTER table tsg_galaxy_v3.monitor_event_local on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+ALTER table tsg_galaxy_v3.monitor_event on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS sip_bye_reason String after sip_bye;
+
 -- tsg_galaxy_v3.security_event_materialized_view
 CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster 
 TO tsg_galaxy_v3.security_event_local
@@ -253,6 +268,7 @@ TO tsg_galaxy_v3.security_event_local
     sip_responder_sdp_content String,
     sip_duration_s Nullable(Int32),
     sip_bye String,
+    sip_bye_reason String,
     rtp_payload_type_c2s Nullable(Int32),
     rtp_payload_type_s2c Nullable(Int32),
     rtp_pcap_path String,
@@ -498,6 +514,7 @@ SELECT
     sip_responder_sdp_content,
     sip_duration_s,
     sip_bye,
+    sip_bye_reason,
     rtp_payload_type_c2s,
     rtp_payload_type_s2c,
     rtp_pcap_path,
@@ -748,6 +765,7 @@ TO tsg_galaxy_v3.monitor_event_local
     sip_responder_sdp_content String,
     sip_duration_s Nullable(Int32),
     sip_bye String,
+    sip_bye_reason String,
     rtp_payload_type_c2s Nullable(Int32),
     rtp_payload_type_s2c Nullable(Int32),
     rtp_pcap_path String,
@@ -993,6 +1011,7 @@ SELECT
     sip_responder_sdp_content,
     sip_duration_s,
     sip_bye,
+    sip_bye_reason,
     rtp_payload_type_c2s,
     rtp_payload_type_s2c,
     rtp_pcap_path,

tsg_olap/upgrade/TSG-24.08/groot_stream/single-cluster-examples/etl_traffic_sketch_metric_kafka_to_clickhouse

@@ -16,7 +16,7 @@ sources:
       format: json
 
 processing_pipelines:
-  pre_etl_processor: # [object] Processing Pipeline
+  etl_processor: # [object] Processing Pipeline
     type: projection
     remove_fields:
     output_fields:
@@ -26,72 +26,7 @@ processing_pipelines:
         output_fields: [ recv_time ]
         parameters:
           precision: seconds
-          interval: 300     
+          interval: 60
-
-  aggregate_processor:
-    type: aggregate
-    group_by_fields: [vsys_id,device_id,device_group,data_center,ip_protocol,direction,client_ip,server_ip,server_domain,app,recv_time]
-    window_type: tumbling_processing_time  # tumbling_event_time,sliding_processing_time,sliding_event_time
-    window_size: 300
-    functions:
-      - function: NUMBER_SUM
-        lookup_fields: [ sessions ]
-      - function: NUMBER_SUM
-        lookup_fields: [ bytes ]
-      - function: NUMBER_SUM
-        lookup_fields: [ sent_bytes ]
-      - function: NUMBER_SUM
-        lookup_fields: [ received_bytes ]
-      - function: NUMBER_SUM
-        lookup_fields: [ pkts ]
-      - function: NUMBER_SUM
-        lookup_fields: [ sent_pkts ]
-      - function: NUMBER_SUM
-        lookup_fields: [ received_pkts ]
-      - function: NUMBER_SUM
-        lookup_fields: [ asymmetric_c2s_flows ]
-      - function: NUMBER_SUM
-        lookup_fields: [ asymmetric_s2c_flows ]
-      - function: NUMBER_SUM
-        lookup_fields: [ c2s_fragments ]
-      - function: NUMBER_SUM
-        lookup_fields: [ s2c_fragments ]
-      - function: NUMBER_SUM
-        lookup_fields: [ c2s_tcp_lost_bytes ]
-      - function: NUMBER_SUM
-        lookup_fields: [ s2c_tcp_lost_bytes ]
-      - function: NUMBER_SUM
-        lookup_fields: [ c2s_tcp_retransmitted_pkts ]
-      - function: NUMBER_SUM
-        lookup_fields: [ s2c_tcp_retransmitted_pkts ]
-      - function: FIRST_VALUE
-        lookup_fields: [ client_country ]
-      - function: FIRST_VALUE
-        lookup_fields: [ server_country ]
-      - function: FIRST_VALUE
-        lookup_fields: [ client_asn ]
-      - function: FIRST_VALUE
-        lookup_fields: [ server_asn ]
-      - function: FIRST_VALUE
-        lookup_fields: [ server_fqdn ]
-      - function: FIRST_VALUE
-        lookup_fields: [ app_category ]
-      - function: FIRST_VALUE
-        lookup_fields: [ c2s_ttl ]
-      - function: FIRST_VALUE
-        lookup_fields: [ s2c_ttl ]
-      - function: FIRST_VALUE
-        lookup_fields: [ c2s_link_id ]
-      - function: FIRST_VALUE
-        lookup_fields: [ s2c_link_id ]
-
-
-
-  post_etl_processor: # [object] Processing Pipeline
-    type: projection
-    remove_fields:
-    output_fields:
-    functions: # [array of object] Function List
       - function: EVAL
         output_fields: [ internal_ip ]
         parameters:
@@ -100,7 +35,6 @@ processing_pipelines:
         output_fields: [ external_ip ]
         parameters:
           value_expression:  'direction=Outbound? server_ip : client_ip'
-
       - function: SNOWFLAKE_ID
         lookup_fields: [ '' ]
         output_fields: [ log_id ]
@@ -129,14 +63,9 @@ application:
     pipeline:
       object-reuse: true # [boolean] Object Reuse, default is false
   topology:
-  topology:
   - name: kafka_source
-    downstream: [pre_etl_processor]
+    downstream: [etl_processor]
-  - name: pre_etl_processor
+  - name: etl_processor
-    downstream: [aggregate_processor]
-  - name: aggregate_processor
-    downstream: [post_etl_processor]
-  - name: post_etl_processor
     downstream: [clickhouse_sink]
   - name: clickhouse_sink
 

tsg_olap/upgrade/TSG-24.08/groot_stream/templates/traffic_sketch_metric.yaml.j2

@@ -16,7 +16,7 @@ sources:
       format: json
 
 processing_pipelines:
-  pre_etl_processor: # [object] Processing Pipeline
+  etl_processor: # [object] Processing Pipeline
     type: projection
     remove_fields:
     output_fields:
@@ -58,7 +58,7 @@ sinks:
       kafka.compression.type: snappy
       kafka.security.protocol: SASL_PLAINTEXT
       kafka.sasl.mechanism: PLAIN
-      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
       format: json
       json.ignore.parse.errors: false
       log.failures.only: true

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_eml_file_chunk_combiner

@@ -0,0 +1,50 @@
+flink.job.name=agg_traffic_eml_file_chunk_combiner
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker=192.168.44.12:9092
+source.kafka.topic=TRAFFIC-EML-FILE-STREAM-RECORD
+source.kafka.group.id=agg_traffic_eml_file_chunk_combiner_1
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library=/opt/tsg/olap/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism=1
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism=1
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint=192.168.44.12:8186
+sink.hos.bucket=traffic_eml_file_bucket
+sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_http_file_chunk_combiner

@@ -0,0 +1,50 @@
+flink.job.name=agg_traffic_http_file_chunk_combiner
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker=192.168.44.12:9092
+source.kafka.topic=TRAFFIC-HTTP-FILE-STREAM-RECORD
+source.kafka.group.id=agg_traffic_http_file_chunk_combiner_1
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library=/opt/tsg/olap/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism=3
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism=3
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint=192.168.44.12:8186
+sink.hos.bucket=traffic_http_file_bucket
+sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_policy_capture_file_chunk_combiner

@@ -0,0 +1,50 @@
+flink.job.name=agg_traffic_policy_capture_file_chunk_combiner
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker=192.168.44.12:9092
+source.kafka.topic=TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD
+source.kafka.group.id=agg_traffic_policy_capture_file_chunk_combiner_1
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library=/opt/tsg/olap/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism=3
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism=3
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint=192.168.44.12:8186
+sink.hos.bucket=traffic_policy_capture_file_bucket
+sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/config/agg_traffic_rtp_file_chunk_combiner

@@ -0,0 +1,50 @@
+flink.job.name=agg_traffic_rtp_file_chunk_combiner
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker=192.168.44.12:9092
+source.kafka.topic=TRAFFIC-RTP-FILE-STREAM-RECORD
+source.kafka.group.id=agg_traffic_rtp_file_chunk_combiner_1
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library=/opt/tsg/olap/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism=3
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism=3
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint=192.168.44.12:8186
+sink.hos.bucket=traffic_rtp_file_bucket
+sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_eml_file_chunk_combiner.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="1"
+export TASK_MODE="yarn-session"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size=1024m
+-Dtaskmanager.numberOfTaskSlots=1
+-Dtaskmanager.memory.framework.off-heap.size=256m
+-Dtaskmanager.memory.jvm-metaspace.size=128m
+"
+

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_http_file_chunk_combiner.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="3"
+export TASK_MODE="yarn-session"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size=2048m
+-Dtaskmanager.numberOfTaskSlots=3
+-Dtaskmanager.memory.framework.off-heap.size=256m
+-Dtaskmanager.memory.jvm-metaspace.size=128m
+"
+

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_policy_capture_file_chunk_combiner.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="3"
+export TASK_MODE="yarn-session"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size=2048m
+-Dtaskmanager.numberOfTaskSlots=3
+-Dtaskmanager.memory.framework.off-heap.size=256m
+-Dtaskmanager.memory.jvm-metaspace.size=128m
+"
+

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/单机/env/agg_traffic_rtp_file_chunk_combiner.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="3"
+export TASK_MODE="yarn-session"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size=2048m
+-Dtaskmanager.numberOfTaskSlots=3
+-Dtaskmanager.memory.framework.off-heap.size=256m
+-Dtaskmanager.memory.jvm-metaspace.size=128m
+"
+

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_eml_file_chunk_combiner

@@ -0,0 +1,50 @@
+flink.job.name=agg_traffic_eml_file_chunk_combiner
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
+source.kafka.topic=TRAFFIC-EML-FILE-STREAM-RECORD
+source.kafka.group.id=agg_traffic_eml_file_chunk_combiner_1
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library=/opt/tsg/olap/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism=1
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism=1
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186
+sink.hos.bucket=traffic_eml_file_bucket
+sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_http_file_chunk_combiner

@@ -0,0 +1,50 @@
+flink.job.name=agg_traffic_http_file_chunk_combiner
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
+source.kafka.topic=TRAFFIC-HTTP-FILE-STREAM-RECORD
+source.kafka.group.id=agg_traffic_http_file_chunk_combiner_1
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library=/opt/tsg/olap/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism=3
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism=3
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186
+sink.hos.bucket=traffic_http_file_bucket
+sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_policy_capture_file_chunk_combiner

@@ -0,0 +1,50 @@
+flink.job.name=agg_traffic_policy_capture_file_chunk_combiner
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
+source.kafka.topic=TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD
+source.kafka.group.id=agg_traffic_policy_capture_file_chunk_combiner_1
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library=/opt/tsg/olap/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism=3
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism=3
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186
+sink.hos.bucket=traffic_policy_capture_file_bucket
+sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/config/agg_traffic_rtp_file_chunk_combiner

@@ -0,0 +1,50 @@
+flink.job.name=agg_traffic_rtp_file_chunk_combiner
+
+#kafka source配置
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
+source.kafka.topic=TRAFFIC-RTP-FILE-STREAM-RECORD
+source.kafka.group.id=agg_traffic_rtp_file_chunk_combiner_1
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=olap
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2024
+#SSL需要
+source.kafka.tools.library=/opt/tsg/olap/topology/data/
+
+map.filter.expression=FileChunk.offset <= 1073741824
+
+#窗口相关配置
+combiner.window.parallelism=3
+#窗口大小，单位秒
+combiner.window.size=10
+
+#sink相关参数
+sink.parallelism=3
+#可选hos、oss、hbase
+sink.type=hos
+sink.async=false
+
+#hos sink相关配置
+#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
+sink.hos.endpoint=192.168.44.11:8186,192.168.44.14:8186
+sink.hos.bucket=traffic_rtp_file_bucket
+sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
+sink.hos.batch.size=1048576
+sink.hos.batch.interval.ms=10000
+
+#http相关配置
+sink.http.client.retries.number=3
+sink.http.client.max.total=20
+sink.http.client.max.per.route=10
+sink.http.client.connect.timeout.ms=10000
+sink.http.client.request.timeout.ms=10000
+sink.http.client.socket.timeout.ms=60000

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_eml_file_chunk_combiner.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="1"
+export TASK_MODE="yarn-per-job"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size=1024m
+-Dtaskmanager.numberOfTaskSlots=1
+-Dtaskmanager.memory.framework.off-heap.size=256m
+-Dtaskmanager.memory.jvm-metaspace.size=128m
+"
+

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_http_file_chunk_combiner.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="3"
+export TASK_MODE="yarn-per-job"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size=2048m
+-Dtaskmanager.numberOfTaskSlots=3
+-Dtaskmanager.memory.framework.off-heap.size=256m
+-Dtaskmanager.memory.jvm-metaspace.size=128m
+"
+

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_policy_capture_file_chunk_combiner.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="3"
+export TASK_MODE="yarn-per-job"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size=2048m
+-Dtaskmanager.numberOfTaskSlots=3
+-Dtaskmanager.memory.framework.off-heap.size=256m
+-Dtaskmanager.memory.jvm-metaspace.size=128m
+"
+

tsg_olap/upgrade/TSG-24.09/file-chunk-combiner/集群/env/agg_traffic_rtp_file_chunk_combiner.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="3"
+export TASK_MODE="yarn-per-job"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size=2048m
+-Dtaskmanager.numberOfTaskSlots=3
+-Dtaskmanager.memory.framework.off-heap.size=256m
+-Dtaskmanager.memory.jvm-metaspace.size=128m
+"
+

tsg_olap/upgrade/TSG-24.09/groot_stream/templates/proxy_event.yaml.j2

@@ -0,0 +1,153 @@
+sources:
+  kafka_source:
+    type: kafka
+    properties:
+      topic: PROXY-EVENT
+      kafka.bootstrap.servers: {{ kafka_source_servers }}
+      kafka.client.id: PROXY-EVENT
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      kafka.group.id: {{ kafka_source_group_id }}
+      kafka.auto.offset.reset: latest
+      format: json
+      json.ignore.parse.errors: false
+
+processing_pipelines:
+  etl_processor:
+    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    functions:
+    - function: SNOWFLAKE_ID
+      lookup_fields: ['']
+      output_fields: [log_id]
+      parameters:
+        data_center_id_num: {{ data_center_id_num }}
+
+    - function: JSON_EXTRACT
+      lookup_fields: [device_tag]
+      output_fields: [data_center]
+      filter:
+      parameters:
+        value_expression: $.tags[?(@.tag=='data_center')][0].value
+
+    - function: JSON_EXTRACT
+      lookup_fields: [device_tag]
+      output_fields: [device_group]
+      filter:
+      parameters:
+        value_expression: $.tags[?(@.tag=='device_group')][0].value
+
+    - function: UNIX_TIMESTAMP_CONVERTER
+      lookup_fields: [__timestamp]
+      output_fields: [recv_time]
+      parameters:
+        precision: seconds
+
+    - function: EVAL
+      output_fields: [ingestion_time]
+      parameters:
+        value_expression: recv_time
+
+    - function: DOMAIN
+      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
+      output_fields: [server_domain]
+      parameters:
+        option: FIRST_SIGNIFICANT_SUBDOMAIN
+
+    - function: BASE64_DECODE_TO_STRING
+      output_fields: [mail_subject]
+      parameters:
+        value_field: mail_subject
+        charset_field: mail_subject_charset
+
+    - function: BASE64_DECODE_TO_STRING
+      output_fields: [mail_attachment_name]
+      parameters:
+        value_field: mail_attachment_name
+        charset_field: mail_attachment_name_charset
+
+    - function: PATH_COMBINE
+      lookup_fields: [rtp_pcap_path]
+      output_fields: [rtp_pcap_path]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path]
+
+    - function: PATH_COMBINE
+      lookup_fields: [http_request_body]
+      output_fields: [http_request_body]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body]
+
+    - function: PATH_COMBINE
+      lookup_fields: [http_response_body]
+      output_fields: [http_response_body]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body]
+
+    - function: PATH_COMBINE
+      lookup_fields: [mail_eml_file]
+      output_fields: [mail_eml_file]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file]
+
+    - function: PATH_COMBINE
+      lookup_fields: [packet_capture_file]
+      output_fields: [packet_capture_file]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
+
+
+    - function: CURRENT_UNIX_TIMESTAMP
+      output_fields: [ processing_time ]
+      parameters:
+        precision: seconds
+
+sinks:
+  kafka_sink:
+    type: kafka
+    properties:
+      topic: PROXY-EVENT
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.client.id: PROXY-EVENT
+      kafka.retries: 0
+      kafka.linger.ms: 10
+      kafka.request.timeout.ms: 30000
+      kafka.batch.size: 262144
+      kafka.buffer.memory: 134217728
+      kafka.max.request.size: 10485760
+      kafka.compression.type: snappy
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      format: json
+      json.ignore.parse.errors: false
+      log.failures.only: true
+
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_sink_host }}
+      table: tsg_galaxy_v3.proxy_event_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: e54c9568586180eede1506eecf3574e9
+      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
+      connection.connect_timeout: 30
+      connection.query_timeout: 300
+
+application:
+  env: 
+    name: {{ job_name }}
+    shade.identifier: aes
+    pipeline:
+      object-reuse: true
+    properties:
+      hos.bucket.name.rtp_file: traffic_rtp_file_bucket
+      hos.bucket.name.http_file: traffic_http_file_bucket
+      hos.bucket.name.eml_file: traffic_eml_file_bucket
+      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
+  {{ topology }}
+

tsg_olap/upgrade/TSG-24.09/groot_stream/templates/session_record.yaml.j2

@@ -0,0 +1,151 @@
+sources:
+  kafka_source:
+    type: kafka
+    properties:
+      topic: SESSION-RECORD
+      kafka.bootstrap.servers: {{ kafka_source_servers }}
+      kafka.client.id: SESSION-RECORD
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      kafka.group.id: {{ kafka_source_group_id }}
+      kafka.auto.offset.reset: latest
+      format: json
+      json.ignore.parse.errors: false
+
+processing_pipelines:
+  etl_processor:
+    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    functions:
+    - function: SNOWFLAKE_ID
+      lookup_fields: ['']
+      output_fields: [log_id]
+      parameters:
+        data_center_id_num: {{ data_center_id_num }}
+
+    - function: JSON_EXTRACT
+      lookup_fields: [device_tag]
+      output_fields: [data_center]
+      filter:
+      parameters:
+        value_expression: $.tags[?(@.tag=='data_center')][0].value
+
+    - function: JSON_EXTRACT
+      lookup_fields: [device_tag]
+      output_fields: [device_group]
+      filter:
+      parameters:
+        value_expression: $.tags[?(@.tag=='device_group')][0].value
+
+    - function: UNIX_TIMESTAMP_CONVERTER
+      lookup_fields: [__timestamp]
+      output_fields: [recv_time]
+      parameters:
+        precision: seconds
+
+    - function: EVAL
+      output_fields: [ingestion_time]
+      parameters:
+        value_expression: recv_time
+
+    - function: DOMAIN
+      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
+      output_fields: [server_domain]
+      parameters:
+        option: FIRST_SIGNIFICANT_SUBDOMAIN
+
+    - function: BASE64_DECODE_TO_STRING
+      output_fields: [mail_subject]
+      parameters:
+        value_field: mail_subject
+        charset_field: mail_subject_charset
+
+    - function: BASE64_DECODE_TO_STRING
+      output_fields: [mail_attachment_name]
+      parameters:
+        value_field: mail_attachment_name
+        charset_field: mail_attachment_name_charset
+
+    - function: PATH_COMBINE
+      lookup_fields: [rtp_pcap_path]
+      output_fields: [rtp_pcap_path]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path]
+
+    - function: PATH_COMBINE
+      lookup_fields: [http_request_body]
+      output_fields: [http_request_body]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body]
+
+    - function: PATH_COMBINE
+      lookup_fields: [http_response_body]
+      output_fields: [http_response_body]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body]
+
+    - function: PATH_COMBINE
+      lookup_fields: [mail_eml_file]
+      output_fields: [mail_eml_file]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file]
+
+    - function: PATH_COMBINE
+      lookup_fields: [packet_capture_file]
+      output_fields: [packet_capture_file]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
+
+    - function: CURRENT_UNIX_TIMESTAMP
+      output_fields: [ processing_time ]
+      parameters:
+        precision: seconds
+
+sinks:
+  kafka_sink:
+    type: kafka
+    properties:
+      topic: SESSION-RECORD
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.client.id: SESSION-RECORD
+      kafka.retries: 0
+      kafka.linger.ms: 10
+      kafka.request.timeout.ms: 30000
+      kafka.batch.size: 262144
+      kafka.buffer.memory: 134217728
+      kafka.max.request.size: 10485760
+      kafka.compression.type: snappy
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      format: json
+      json.ignore.parse.errors: false
+      log.failures.only: true
+
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_sink_host }}
+      table: tsg_galaxy_v3.session_record_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: e54c9568586180eede1506eecf3574e9
+      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
+      connection.connect_timeout: 30
+      connection.query_timeout: 300
+
+application:
+  env:
+    name: {{ job_name }}
+    shade.identifier: aes
+    pipeline:
+      object-reuse: true
+    properties:
+      hos.bucket.name.rtp_file: traffic_rtp_file_bucket
+      hos.bucket.name.http_file: traffic_http_file_bucket
+      hos.bucket.name.eml_file: traffic_eml_file_bucket
+      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
+  {{ topology }}

tsg_olap/upgrade/TSG-24.09/groot_stream/templates/transaction_record.yaml.j2

@@ -0,0 +1,151 @@
+sources:
+  kafka_source:
+    type: kafka
+    properties:
+      topic: TRANSACTION-RECORD
+      kafka.bootstrap.servers: {{ kafka_source_servers }}
+      kafka.client.id: TRANSACTION-RECORD
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      kafka.group.id: {{ kafka_source_group_id }}
+      kafka.auto.offset.reset: latest
+      format: json
+      json.ignore.parse.errors: false
+
+processing_pipelines:
+  etl_processor:
+    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    functions:
+    - function: SNOWFLAKE_ID
+      lookup_fields: ['']
+      output_fields: [log_id]
+      parameters:
+        data_center_id_num: {{ data_center_id_num }}
+
+    - function: JSON_EXTRACT
+      lookup_fields: [device_tag]
+      output_fields: [data_center]
+      filter:
+      parameters:
+        value_expression: $.tags[?(@.tag=='data_center')][0].value
+
+    - function: JSON_EXTRACT
+      lookup_fields: [device_tag]
+      output_fields: [device_group]
+      filter:
+      parameters:
+        value_expression: $.tags[?(@.tag=='device_group')][0].value
+
+    - function: UNIX_TIMESTAMP_CONVERTER
+      lookup_fields: [__timestamp]
+      output_fields: [recv_time]
+      parameters:
+        precision: seconds
+
+    - function: EVAL
+      output_fields: [ingestion_time]
+      parameters:
+        value_expression: recv_time
+
+    - function: DOMAIN
+      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
+      output_fields: [server_domain]
+      parameters:
+        option: FIRST_SIGNIFICANT_SUBDOMAIN
+
+    - function: BASE64_DECODE_TO_STRING
+      output_fields: [mail_subject]
+      parameters:
+        value_field: mail_subject
+        charset_field: mail_subject_charset
+
+    - function: BASE64_DECODE_TO_STRING
+      output_fields: [mail_attachment_name]
+      parameters:
+        value_field: mail_attachment_name
+        charset_field: mail_attachment_name_charset
+
+    - function: PATH_COMBINE
+      lookup_fields: [rtp_pcap_path]
+      output_fields: [rtp_pcap_path]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path]
+
+    - function: PATH_COMBINE
+      lookup_fields: [http_request_body]
+      output_fields: [http_request_body]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body]
+
+    - function: PATH_COMBINE
+      lookup_fields: [http_response_body]
+      output_fields: [http_response_body]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body]
+
+    - function: PATH_COMBINE
+      lookup_fields: [mail_eml_file]
+      output_fields: [mail_eml_file]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file]
+
+    - function: PATH_COMBINE
+      lookup_fields: [packet_capture_file]
+      output_fields: [packet_capture_file]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
+
+    - function: CURRENT_UNIX_TIMESTAMP
+      output_fields: [ processing_time ]
+      parameters:
+        precision: seconds
+
+sinks:
+  kafka_sink:
+    type: kafka
+    properties:
+      topic: TRANSACTION-RECORD
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.client.id: TRANSACTION-RECORD
+      kafka.retries: 0
+      kafka.linger.ms: 10
+      kafka.request.timeout.ms: 30000
+      kafka.batch.size: 262144
+      kafka.buffer.memory: 134217728
+      kafka.max.request.size: 10485760
+      kafka.compression.type: snappy
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      format: json
+      json.ignore.parse.errors: false
+      log.failures.only: true
+
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_sink_host }}
+      table: tsg_galaxy_v3.transaction_record_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: e54c9568586180eede1506eecf3574e9
+      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
+      connection.connect_timeout: 30
+      connection.query_timeout: 300
+
+application:
+  env: 
+    name: {{ job_name }}
+    shade.identifier: aes
+    pipeline:
+      object-reuse: true
+    properties:
+      hos.bucket.name.rtp_file: traffic_rtp_file_bucket
+      hos.bucket.name.http_file: traffic_http_file_bucket
+      hos.bucket.name.eml_file: traffic_eml_file_bucket
+      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
+  {{ topology }}

tsg_olap/upgrade/TSG-24.09/groot_stream/templates/voip_record.yaml.j2

@@ -0,0 +1,151 @@
+sources:
+  kafka_source:
+    type: kafka
+    properties:
+      topic: VOIP-CONVERSATION-RECORD
+      kafka.bootstrap.servers: {{ kafka_source_servers }}
+      kafka.client.id: VOIP-CONVERSATION-RECORD
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      kafka.group.id: {{ kafka_source_group_id }}
+      kafka.auto.offset.reset: latest
+      format: json
+      json.ignore.parse.errors: false
+
+processing_pipelines:
+  etl_processor:
+    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    functions:
+    - function: SNOWFLAKE_ID
+      lookup_fields: ['']
+      output_fields: [log_id]
+      parameters:
+        data_center_id_num: {{ data_center_id_num }}
+
+    - function: JSON_EXTRACT
+      lookup_fields: [device_tag]
+      output_fields: [data_center]
+      filter:
+      parameters:
+        value_expression: $.tags[?(@.tag=='data_center')][0].value
+
+    - function: JSON_EXTRACT
+      lookup_fields: [device_tag]
+      output_fields: [device_group]
+      filter:
+      parameters:
+        value_expression: $.tags[?(@.tag=='device_group')][0].value
+
+    - function: UNIX_TIMESTAMP_CONVERTER
+      lookup_fields: [__timestamp]
+      output_fields: [recv_time]
+      parameters:
+        precision: seconds
+
+    - function: EVAL
+      output_fields: [ingestion_time]
+      parameters:
+        value_expression: recv_time
+
+    - function: DOMAIN
+      lookup_fields: [http_host, ssl_sni, dtls_sni, quic_sni]
+      output_fields: [server_domain]
+      parameters:
+        option: FIRST_SIGNIFICANT_SUBDOMAIN
+
+    - function: BASE64_DECODE_TO_STRING
+      output_fields: [mail_subject]
+      parameters:
+        value_field: mail_subject
+        charset_field: mail_subject_charset
+
+    - function: BASE64_DECODE_TO_STRING
+      output_fields: [mail_attachment_name]
+      parameters:
+        value_field: mail_attachment_name
+        charset_field: mail_attachment_name_charset
+
+    - function: PATH_COMBINE
+      lookup_fields: [rtp_pcap_path]
+      output_fields: [rtp_pcap_path]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.rtp_file, rtp_pcap_path]
+
+    - function: PATH_COMBINE
+      lookup_fields: [http_request_body]
+      output_fields: [http_request_body]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.http_file, http_request_body]
+
+    - function: PATH_COMBINE
+      lookup_fields: [http_response_body]
+      output_fields: [http_response_body]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.http_file, http_response_body]
+
+    - function: PATH_COMBINE
+      lookup_fields: [mail_eml_file]
+      output_fields: [mail_eml_file]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.eml_file, mail_eml_file]
+
+    - function: PATH_COMBINE
+      lookup_fields: [packet_capture_file]
+      output_fields: [packet_capture_file]
+      parameters:
+        path: [props.hos.path, props.hos.bucket.name.policy_capture_file, packet_capture_file]
+
+    - function: CURRENT_UNIX_TIMESTAMP
+      output_fields: [ processing_time ]
+      parameters:
+        precision: seconds
+
+sinks:
+  kafka_sink:
+    type: kafka
+    properties:
+      topic: VOIP-CONVERSATION-RECORD
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.client.id: VOIP-CONVERSATION-RECORD
+      kafka.retries: 0
+      kafka.linger.ms: 10
+      kafka.request.timeout.ms: 30000
+      kafka.batch.size: 262144
+      kafka.buffer.memory: 134217728
+      kafka.max.request.size: 10485760
+      kafka.compression.type: snappy
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252
+      format: json
+      json.ignore.parse.errors: false
+      log.failures.only: true
+
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_sink_host }}
+      table: tsg_galaxy_v3.voip_record_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: e54c9568586180eede1506eecf3574e9
+      connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
+      connection.connect_timeout: 30
+      connection.query_timeout: 300
+
+application:
+  env: 
+    name: {{ job_name }}
+    shade.identifier: aes
+    pipeline:
+      object-reuse: true
+    properties:
+      hos.bucket.name.rtp_file: traffic_rtp_file_bucket
+      hos.bucket.name.http_file: traffic_http_file_bucket
+      hos.bucket.name.eml_file: traffic_eml_file_bucket
+      hos.bucket.name.policy_capture_file: traffic_policy_capture_file_bucket
+  {{ topology }}

tsg_olap/upgrade/TSG-24.09/hbase/update_hbase_24.09.sh

@@ -1,19 +1,11 @@
+docker exec -i HMaster hbase shell <<EOF
 alter 'troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}}
 alter 'index_time_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}}
 alter 'index_filename_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}}
 alter 'index_partfile_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}}
-
 alter 'assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}}
 alter 'index_time_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}}
 alter 'index_filename_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}}
 alter 'index_partfile_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}}
 
-alter 'knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}}
+EOF
-alter 'index_time_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}}
-alter 'index_filename_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}}
-alter 'index_partfile_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}}
-
-alter 'report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}}
-alter 'index_time_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}}
-alter 'index_filename_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}}
-alter 'index_partfile_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}}

tsg_olap/upgrade/TSG-24.09/hos/create_bucket_24.09.sh

@@ -4,5 +4,5 @@ curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_ht
 curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_eml_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1'
 curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/troubleshooting_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1'
 curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/assessment_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-ttl:30' -H 'x-hos-replication:1'
-curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/knowledge_base_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:2'
+curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/knowledge_base_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:单机为1集群为2'
-curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/report_snapshot_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:2'
+curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/report_snapshot_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:单机为1集群为2'

tsg_olap/upgrade/TSG-24.09/hos/hosutil/config.properties

@@ -0,0 +1,21 @@
+qgw.server.addr=http://192.168.44.67:9999
+hos.server.addr=http://192.168.44.67:9098
+hos.token=c21f969b5f03d33d43e04f8f136e7682
+kafka.server=192.168.44.11:9092
+#延迟时间，校验多少秒之前的文件，单位秒
+check.time.delay=180
+hos.traffic.buckets=traffic_policy_capture_file_bucket,traffic_rtp_file_bucket,traffic_http_file_bucket,traffic_eml_file_bucket
+kafka.traffic.topics=TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD,TRAFFIC-RTP-FILE-STREAM-RECORD,TRAFFIC-HTTP-FILE-STREAM-RECORD,TRAFFIC-EML-FILE-STREAM-RECORD
+kafka.troubleshooting.topic=TROUBLESHOOTING-FILE-STREAM-RECORD
+file.chunk.combiner.window.time=15000
+traffic.file.count=10
+threads=1
+max.threads=10
+print.out.interval=1000
+http.max.total=100
+http.default.max.per.route=100
+http.connect.timeout=5000
+http.connection.request.timeout=10000
+http.socket.timeout=-1
+hos.log.types=security_event,monitor_event,proxy_event,session_record,voip_record,assessment_event,transaction_record,troubleshooting
+hos.log.types.file.types.url.fields=security_event:http-http_response_body&http_request_body,pcap-packet_capture_file&rtp_pcap_path,eml-mail_eml_file;proxy_event:http-http_response_body&http_request_body;session_record:http-http_response_body&http_request_body,pcap-packet_capture_file&rtp_pcap_path,eml-mail_eml_file;voip_record:pcap-rtp_pcap_path;assessment_event:other-assessment_file;transaction_record:http-http_response_body&http_request_body,eml-mail_eml_file;monitor_event:http-http_response_body&http_request_body,pcap-packet_capture_file&rtp_pcap_path,eml-mail_eml_file

tsg_olap/upgrade/TSG-24.09/hos/hosutil/hosutil.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+version="1.4"
+jar="galaxy-hos-util-$version.jar"
+
+usage() {
+  cat <<EOF
+
+Usage: ./hosutil.sh [command] [-h] [options...]
+
+Available commands:
+  download      Download individual or batch files
+  upload        Upload individual or batch files
+  check         Check file availability
+  combiner      Verify if the file-chunk-combiner data stream is correct
+  version       Print the version
+
+Options for 'download' command:
+  -b, --bucket        The bucket to access.
+  -d, --directory     Directory to save files. If not exists, will be created. Default is ./download/.
+  -k, --keys          Files to download. Can be a single or multiple files separated by commas.
+  -p, --prefix        Prefix for batch downloading files based on file name.
+  -s, --start_time    Start time in UTC format (yyyyMMdd, yyyy-MM-dd, yyyyMMddHHmmss). Default is the previous day's time.
+  -e, --end_time      End time in UTC format (yyyyMMdd, yyyy-MM-dd, yyyyMMddHHmmss). Default is current time.
+  -c, --count         Number of files to download. Default is 1000, maximum is 100000.
+  -t, --threads       Number of threads. Default is 1, maximum is 10.
+
+Options for 'upload' command:
+  -b, --bucket        The bucket to access.
+  -d, --directory     Directory where files to upload are located. Default is ./upload/.
+  -t, --threads       Number of threads. Default is 1, maximum is 10.
+
+Options for 'check' command:
+  -s, --start_time    Start time in UTC format (yyyyMMdd, yyyy-MM-dd, yyyyMMddHHmmss). Default is the previous day's time.
+  -e, --end_time      End time in UTC format (yyyyMMdd, yyyy-MM-dd, yyyyMMddHHmmss). Default is current time.
+  -c, --count         Number of logs to evaluate. Default is 1000, maximum is 100000.
+  -d, --data_center   Specify the data centers to evaluate, separated by commas. If not specified, all data centers are evaluated.
+  -l, --log_type      Specify the logs to evaluate, separated by commas. If not specified, all logs are evaluated.
+                      Supported logs: security_event, monitor_event, proxy_event, session_record, voip_record, assessment_event, transaction_record, troubleshooting.
+  -f, --file_type     Specify file types. If not specified, all types are evaluated. Supported types: eml, http, pcap, other.
+                      Only session_record, security_event, monitor_event, transaction_record support multiple types.
+  -t --threads        Number of threads. Default is 1, maximum is 10.
+
+Options for 'combiner' command:
+  -j, --job           Job to verify. Options: traffic, troubleshooting. Default is traffic.(Troubleshooting job removed in version 24.05)
+
+EOF
+}
+
+# 初始化默认值
+bucket=""
+directory=""
+keys=""
+prefix=""
+start_time=""
+end_time=""
+count=1000
+threads=1
+log_type=""
+file_type=""
+data_center=""
+job_name="traffic"
+
+# 检查必填参数
+check_required() {
+  case "$operation" in
+    download|upload)
+      if [ -z "$bucket" ]; then
+        echo "Error: bucket is required for $operation."
+        exit 1
+      fi
+      ;;
+    *)
+      # 对于其他操作，不需要检查特定参数
+      ;;
+  esac
+}
+
+# 下载函数
+download() {
+  directory=${directory:-"./download/"}
+  check_required
+  java -jar $jar download $bucket $directory keys=$keys prefix=$prefix max_keys=$count time_range=$start_time/$end_time thread_num=$threads
+}
+
+# 上传函数
+upload() {
+  directory=${directory:-"./upload/"}
+  check_required
+  java -jar $jar upload $bucket $directory thread_num=$threads
+}
+
+# 检查函数
+check() {
+  java -jar $jar check data_center=$data_center log_type=$log_type file_type=$file_type max_logs=$count time_range=$start_time/$end_time thread_num=$threads
+}
+
+# 合并器函数
+combiner() {
+  java -jar $jar combiner $job_name
+}
+
+# 主操作流程
+if [ $# -eq 0 ];then
+  usage
+  exit 0
+fi
+
+operation=$1
+shift
+while getopts ":h:b:d:k:p:s:e:c:t:l:f:j:" opt; do
+  case $opt in
+    h) usage; exit 0 ;;
+    b) bucket=$OPTARG ;;
+    d) if [ "$operation" == "check" ]; then data_center=$OPTARG; else directory=$OPTARG; fi ;;
+    k) keys=$OPTARG ;;
+    p) prefix=$OPTARG ;;
+    s) start_time=$OPTARG ;;
+    e) end_Time=$OPTARG ;;
+    c) count=$OPTARG ;;
+    t) threads=$OPTARG ;;
+    l) log_type=$OPTARG ;;
+    f) file_type=$OPTARG ;;
+    j) job_name=$OPTARG ;;
+    \?) echo "Invalid option: -$OPTARG" >&2; usage; exit 1 ;;
+    :) echo "Option -$OPTARG requires an argument" >&2; usage; exit 1 ;;
+  esac
+done
+
+case "$operation" in
+  download) download ;;
+  upload) upload ;;
+  check) check ;;
+  combiner) combiner ;;
+  version) echo $version ;;
+  *) usage; exit 1 ;;
+esac
+