diff --git a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql index 818d4ae..41347dd 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_TSG_建表语句.sql @@ -150,6 +150,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -277,8 +279,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -392,6 +394,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -519,8 +523,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -632,6 +636,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -759,8 +765,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -871,6 +877,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -998,8 +1006,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -1112,6 +1120,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -1239,8 +1249,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -1351,6 +1361,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -1478,8 +1490,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -1591,6 +1603,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -1718,8 +1732,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -1832,6 +1846,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -1959,8 +1975,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -2071,6 +2087,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -2198,8 +2216,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, ftp_account String, ftp_url String, @@ -2357,8 +2375,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, sip_call_id String, sip_originator_description String, @@ -2451,8 +2469,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, sip_call_id String, sip_originator_description String, @@ -2543,8 +2561,8 @@ mail_bcc String, mail_subject String, mail_subject_charset String, mail_attachment_name String, -mail_attachment_name_charset String, -mail_starttls_flag Nullable(Int32), +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), mail_eml_file String, sip_call_id String, sip_originator_description String, @@ -2593,27 +2611,6 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), client_ip String, client_port Int32, client_os_desc String, @@ -2623,11 +2620,6 @@ client_super_administrative_area String, client_administrative_area String, client_sub_administrative_area String, client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, server_ip String, server_port Int32, server_os_desc String, @@ -2637,16 +2629,7 @@ server_super_administrative_area String, server_administrative_area String, server_sub_administrative_area String, server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -app_extra_info String, -fqdn_category_list Array(Int64), ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), sip_call_id String, sip_originator_description String, sip_responder_description String, @@ -2669,30 +2652,7 @@ rtp_originator_dir Nullable(Int32), sent_pkts Int64, received_pkts Int64, sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_id_list Array(Int64), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String +received_bytes Int64 ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time)) @@ -2722,27 +2682,6 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), client_ip String, client_port Int32, client_os_desc String, @@ -2752,11 +2691,6 @@ client_super_administrative_area String, client_administrative_area String, client_sub_administrative_area String, client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, server_ip String, server_port Int32, server_os_desc String, @@ -2766,16 +2700,7 @@ server_super_administrative_area String, server_administrative_area String, server_sub_administrative_area String, server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -app_extra_info String, -fqdn_category_list Array(Int64), ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), sip_call_id String, sip_originator_description String, sip_responder_description String, @@ -2798,30 +2723,7 @@ rtp_originator_dir Nullable(Int32), sent_pkts Int64, received_pkts Int64, sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_id_list Array(Int64), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String +received_bytes Int64 ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); @@ -2849,27 +2751,6 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, -security_rule_list Array(Int64), -security_action String, -monitor_rule_list Array(Int64), -shaping_rule_list Array(Int64), -proxy_rule_list Array(Int64), -statistics_rule_list Array(Int64), -sc_rule_list Array(Int64), -sc_rsp_raw Array(Int64), -sc_rsp_decrypted Array(Int64), -proxy_action String, -proxy_pinning_status Nullable(Int32), -proxy_intercept_status Nullable(Int32), -proxy_passthrough_reason String, -proxy_client_side_latency_ms Nullable(Int32), -proxy_server_side_latency_ms Nullable(Int32), -proxy_client_side_version String, -proxy_server_side_version String, -proxy_cert_verify Nullable(Int32), -proxy_intercept_error String, -monitor_mirrored_pkts Nullable(Int32), -monitor_mirrored_bytes Nullable(Int32), client_ip String, client_port Int32, client_os_desc String, @@ -2879,11 +2760,6 @@ client_super_administrative_area String, client_administrative_area String, client_sub_administrative_area String, client_asn Nullable(Int64), -subscriber_id String, -imei String, -imsi String, -phone_number String, -apn String, server_ip String, server_port Int32, server_os_desc String, @@ -2893,16 +2769,7 @@ server_super_administrative_area String, server_administrative_area String, server_sub_administrative_area String, server_asn Nullable(Int64), -server_fqdn String, -server_domain String, -app_transition String,  -app LowCardinality(String), -app_debug_info String, -app_content String, -app_extra_info String, -fqdn_category_list Array(Int64), ip_protocol LowCardinality(String), -decoded_path LowCardinality(String), sip_call_id String, sip_originator_description String, sip_responder_description String, @@ -2925,30 +2792,7 @@ rtp_originator_dir Nullable(Int32), sent_pkts Int64, received_pkts Int64, sent_bytes Int64, -received_bytes Int64, -tcp_c2s_ip_fragments Nullable(Int64), -tcp_s2c_ip_fragments Nullable(Int64), -tcp_c2s_lost_bytes Nullable(Int64), -tcp_s2c_lost_bytes Nullable(Int64), -tcp_c2s_o3_pkts Nullable(Int64), -tcp_s2c_o3_pkts Nullable(Int64), -tcp_c2s_rtx_pkts Nullable(Int64), -tcp_s2c_rtx_pkts Nullable(Int64), -tcp_c2s_rtx_bytes Nullable(Int64), -tcp_s2c_rtx_bytes Nullable(Int64), -tcp_rtt_ms Nullable(Int32), -tcp_client_isn Nullable(Int64), -tcp_server_isn Nullable(Int64), -packet_capture_file String, -in_src_mac String, -out_src_mac String, -in_dest_mac String, -out_dest_mac String, -encapsulation String, -dup_traffic_flag Nullable(Int32), -tunnel_id_list Array(Int64), -tunnel_endpoint_a_desc String, -tunnel_endpoint_b_desc String +received_bytes Int64 ) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); @@ -2978,6 +2822,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -3139,6 +2985,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -3297,6 +3145,8 @@ vsys_id Int32, t_vsys_id Int32, flags Int64, flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -3459,6 +3309,8 @@ TO tsg_galaxy_v3.security_event_local t_vsys_id Int32, flags Int64, flags_identify_info String, + c2s_ttl Nullable(Int32), + s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -3697,6 +3549,8 @@ SELECT t_vsys_id, flags, flags_identify_info, + c2s_ttl, + s2c_ttl, security_rule_list, security_action, monitor_rule_list, @@ -3940,6 +3794,8 @@ TO tsg_galaxy_v3.monitor_event_local t_vsys_id Int32, flags Int64, flags_identify_info String, + c2s_ttl Nullable(Int32), + s2c_ttl Nullable(Int32), security_rule_list Array(Int64), security_action String, monitor_rule_list Array(Int64), @@ -4178,6 +4034,8 @@ SELECT t_vsys_id, flags, flags_identify_info, + c2s_ttl, + s2c_ttl, security_rule_list, security_action, monitor_rule_list, diff --git a/Clickhouse最新全量建表语句/Clickhouse_TSG_校验sql.sql b/Clickhouse最新全量建表语句/Clickhouse_TSG_校验sql.sql index b21c871..5eb0ff7 100644 --- a/Clickhouse最新全量建表语句/Clickhouse_TSG_校验sql.sql +++ b/Clickhouse最新全量建表语句/Clickhouse_TSG_校验sql.sql @@ -2,17 +2,17 @@ SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, asses FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time