Merge remote-tracking branch 'origin/master'
This commit is contained in:
zhanghongqing
@@ -3203,6 +3203,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record_local on clus
|
||||
recv_time Int64,
|
||||
vsys_id Int32,
|
||||
timestamp_us UInt64,
|
||||
egress_action Int32,
|
||||
job_id String,
|
||||
sled_ip String,
|
||||
device_group String,
|
||||
@@ -3224,6 +3225,7 @@ CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record on cluster ck
|
||||
recv_time Int64,
|
||||
vsys_id Int32,
|
||||
timestamp_us UInt64,
|
||||
egress_action Int32,
|
||||
job_id String,
|
||||
sled_ip String,
|
||||
device_group String,
|
||||
@@ -3240,4 +3242,101 @@ ENGINE = Distributed('ck_cluster',
|
||||
'tsg_galaxy_v3',
|
||||
'datapath_telemetry_record_local',
|
||||
rand());
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.traffic_sketch_metric_local on cluster ck_cluster
|
||||
(
|
||||
log_id UInt64,
|
||||
recv_time Int64,
|
||||
vsys_id Int64,
|
||||
device_id String,
|
||||
device_group String,
|
||||
data_center String,
|
||||
direction String,
|
||||
ip_protocol String,
|
||||
client_ip String,
|
||||
server_ip String,
|
||||
internal_ip String,
|
||||
external_ip String,
|
||||
client_country String,
|
||||
server_country String,
|
||||
client_asn Nullable(Int64),
|
||||
server_asn Nullable(Int64),
|
||||
server_fqdn String,
|
||||
server_domain String,
|
||||
app String,
|
||||
app_category String,
|
||||
c2s_ttl Nullable(Int32),
|
||||
s2c_ttl Nullable(Int32),
|
||||
c2s_link_id Nullable(Int32),
|
||||
s2c_link_id Nullable(Int32),
|
||||
sessions Int64,
|
||||
bytes Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
pkts Int64,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
asymmetric_c2s_flows Int64,
|
||||
asymmetric_s2c_flows Int64,
|
||||
c2s_fragments Int64,
|
||||
s2c_fragments Int64,
|
||||
c2s_tcp_lost_bytes Int64,
|
||||
s2c_tcp_lost_bytes Int64,
|
||||
c2s_tcp_retransmitted_pkts Int64,
|
||||
s2c_tcp_retransmitted_pkts Int64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(toDate(recv_time))
|
||||
ORDER BY (vsys_id,
|
||||
direction,
|
||||
ip_protocol,
|
||||
app,
|
||||
client_ip,
|
||||
recv_time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.traffic_sketch_metric on cluster ck_cluster
|
||||
(
|
||||
log_id UInt64,
|
||||
recv_time Int64,
|
||||
vsys_id Int64,
|
||||
device_id String,
|
||||
device_group String,
|
||||
data_center String,
|
||||
direction String,
|
||||
ip_protocol String,
|
||||
client_ip String,
|
||||
server_ip String,
|
||||
internal_ip String,
|
||||
external_ip String,
|
||||
client_country String,
|
||||
server_country String,
|
||||
client_asn Nullable(Int64),
|
||||
server_asn Nullable(Int64),
|
||||
server_fqdn String,
|
||||
server_domain String,
|
||||
app String,
|
||||
app_category String,
|
||||
c2s_ttl Nullable(Int32),
|
||||
s2c_ttl Nullable(Int32),
|
||||
c2s_link_id Nullable(Int32),
|
||||
s2c_link_id Nullable(Int32),
|
||||
sessions Int64,
|
||||
bytes Int64,
|
||||
sent_bytes Int64,
|
||||
received_bytes Int64,
|
||||
pkts Int64,
|
||||
sent_pkts Int64,
|
||||
received_pkts Int64,
|
||||
asymmetric_c2s_flows Int64,
|
||||
asymmetric_s2c_flows Int64,
|
||||
c2s_fragments Int64,
|
||||
s2c_fragments Int64,
|
||||
c2s_tcp_lost_bytes Int64,
|
||||
s2c_tcp_lost_bytes Int64,
|
||||
c2s_tcp_retransmitted_pkts Int64,
|
||||
s2c_tcp_retransmitted_pkts Int64
|
||||
)
|
||||
ENGINE = Distributed('ck_cluster',
|
||||
'tsg_galaxy_v3',
|
||||
'traffic_sketch_metric_local',
|
||||
rand());
|
||||
@@ -14,8 +14,9 @@ SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_tim
|
||||
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
|
||||
FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT log_id, recv_time, vsys_id, timestamp_us, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
|
||||
SELECT log_id, recv_time, vsys_id, timestamp_us, egress_action, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
|
||||
FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
SELECT log_id, recv_time, vsys_id, device_id, device_group, data_center, direction, ip_protocol, client_ip, server_ip, internal_ip, external_ip, client_country, server_country, client_asn, server_asn, server_fqdn, server_domain, app, app_category, c2s_ttl, s2c_ttl, c2s_link_id, s2c_link_id, sessions, bytes, sent_bytes, received_bytes, pkts, sent_pkts, received_pkts, asymmetric_c2s_flows, asymmetric_s2c_flows, c2s_fragments, s2c_fragments, c2s_tcp_lost_bytes, s2c_tcp_lost_bytes, c2s_tcp_retransmitted_pkts, s2c_tcp_retransmitted_pkts
|
||||
FROM tsg_galaxy_v3.traffic_sketch_metric where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,22 @@
|
||||
SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
|
||||
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
|
||||
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
|
||||
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
|
||||
FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT log_id, recv_time, vsys_id, timestamp_us, egress_action, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
|
||||
FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT log_id, recv_time, vsys_id, device_id, device_group, data_center, direction, ip_protocol, client_ip, server_ip, internal_ip, external_ip, client_country, server_country, client_asn, server_asn, server_fqdn, server_domain, app, app_category, c2s_ttl, s2c_ttl, c2s_link_id, s2c_link_id, sessions, bytes, sent_bytes, received_bytes, pkts, sent_pkts, received_pkts, asymmetric_c2s_flows, asymmetric_s2c_flows, c2s_fragments, s2c_fragments, c2s_tcp_lost_bytes, s2c_tcp_lost_bytes, c2s_tcp_retransmitted_pkts, s2c_tcp_retransmitted_pkts
|
||||
FROM tsg_galaxy_v3.traffic_sketch_metric where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
set distributed_ddl_task_timeout = 180;
|
||||
|
||||
-- TSG-21759 clickhouse datapath_telemetry_record 新增字段egress_action
|
||||
ALTER table tsg_galaxy_v3.datapath_telemetry_record_local on cluster ck_cluster add column IF NOT EXISTS egress_action Int32 after timestamp_us;
|
||||
ALTER table tsg_galaxy_v3.datapath_telemetry_record on cluster ck_cluster add column IF NOT EXISTS egress_action Int32 after timestamp_us;
|
||||
@@ -0,0 +1,77 @@
|
||||
flink.job.name=agg_traffic_file_chunk_combine
|
||||
|
||||
#kafka source配置
|
||||
#9092为无验证 9095为ssl 9094为sasl
|
||||
source.kafka.broker=192.168.44.12:9092
|
||||
source.kafka.topic=TRAFFIC-FILE-STREAM-RECORD
|
||||
source.kafka.group.id=test
|
||||
#earliest从头开始 latest最新
|
||||
source.kafka.auto.offset.reset=latest
|
||||
source.kafka.session.timeout.ms=60000
|
||||
#每次拉取操作从分区中获取的最大记录数
|
||||
source.kafka.max.poll.records=1000
|
||||
#消费者从单个分区中一次性获取的最大字节数
|
||||
source.kafka.max.partition.fetch.bytes=31457280
|
||||
source.kafka.enable.auto.commit=true
|
||||
#kafka SASL验证用户名
|
||||
source.kafka.user=admin
|
||||
#kafka SASL及SSL验证密码
|
||||
source.kafka.pin=galaxy2019
|
||||
#SSL需要
|
||||
source.kafka.tools.library=/opt/tsg/olap/topology/data/
|
||||
|
||||
map.filter.expression=FileChunk.offset <= 1073741824
|
||||
|
||||
#窗口相关配置
|
||||
combiner.window.parallelism=1
|
||||
#窗口大小，单位秒
|
||||
combiner.window.size=30
|
||||
#combiner.window.enable.last.chunk.trigger=true
|
||||
|
||||
#sink相关参数
|
||||
sink.parallelism=1
|
||||
#可选hos、oss、hbase
|
||||
sink.type=hos
|
||||
sink.async=true
|
||||
#sink.filter.expression=
|
||||
#限流配置
|
||||
#sink.rate.limit.threshold=0
|
||||
#sink.rate.limit.exclusion.expression=FileChunk.fileType == "eml"
|
||||
|
||||
#hos sink相关配置
|
||||
#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
|
||||
sink.hos.endpoint=192.168.44.12:8186
|
||||
sink.hos.bucket=traffic_file_bucket
|
||||
sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
|
||||
#sink.hos.batch.size=1048576
|
||||
#sink.hos.batch.interval.ms=10000
|
||||
#sink.hos.health.check.interval.ms=60000
|
||||
|
||||
#http相关配置
|
||||
sink.http.client.retries.number=3
|
||||
sink.http.client.max.total=20
|
||||
sink.http.client.max.per.route=10
|
||||
sink.http.client.connect.timeout.ms=10000
|
||||
sink.http.client.request.timeout.ms=10000
|
||||
sink.http.client.socket.timeout.ms=60000
|
||||
|
||||
#hbase sink相关配置
|
||||
sink.hbase.zookeeper=192.168.44.12
|
||||
sink.hbase.retries.number=10
|
||||
sink.hbase.rpc.timeout.ms=600000
|
||||
sink.hbase.client.write.buffer=10971520
|
||||
sink.hbase.client.ipc.pool.size=3
|
||||
sink.hbase.batch.size=1048576
|
||||
sink.hbase.batch.interval.ms=10000
|
||||
|
||||
#oss sink相关配置
|
||||
#可以多个地址，ip1:port,ip2:port...
|
||||
#sink.oss.endpoint=192.168.44.12:8186
|
||||
#sink.oss.filter.expression=FileChunk.offset == 0 && FileChunk.lastChunkFlag == 1
|
||||
#sink.oss.cache.time.ms=30000
|
||||
#sink.oss.cache.size=1073741824
|
||||
#文件关联元信息相关配置
|
||||
#source.kafka.file.meta.session.topic=SESSION-RECORD
|
||||
#source.kafka.file.meta.proxy.topic=PROXY-EVENT
|
||||
#source.kafka.file.meta.group.id=file_chunk_combine_1
|
||||
#file.meta.filter.expression=FileChunk.meta.fileId.contains("_9")
|
||||
24
tsg_olap/upgrade/TSG-24.07/hbase/update-hbase-24.07.txt
Normal file
24
tsg_olap/upgrade/TSG-24.07/hbase/update-hbase-24.07.txt
Normal file
@@ -0,0 +1,24 @@
|
||||
alter 'traffic_file_bucket',{METADATA => {'REGION_MEMSTORE_REPLICATION' => 'false', 'hbase.region.replica.wait.for.primary.flush' => 'false','SPLIT_ENABLED' => 'false',}}
|
||||
alter 'index_time_traffic_file_bucket',{METADATA => {'REGION_MEMSTORE_REPLICATION' => 'false', 'hbase.region.replica.wait.for.primary.flush' => 'false','SPLIT_ENABLED' => 'false',}}
|
||||
alter 'index_filename_traffic_file_bucket',{METADATA => {'REGION_MEMSTORE_REPLICATION' => 'false', 'hbase.region.replica.wait.for.primary.flush' => 'false','SPLIT_ENABLED' => 'false',}}
|
||||
alter 'index_partfile_traffic_file_bucket',{METADATA => {'REGION_MEMSTORE_REPLICATION' => 'false', 'hbase.region.replica.wait.for.primary.flush' => 'false','SPLIT_ENABLED' => 'false',}}
|
||||
|
||||
alter 'troubleshooting_file_bucket',{METADATA => {'REGION_MEMSTORE_REPLICATION' => 'false', 'hbase.region.replica.wait.for.primary.flush' => 'false','SPLIT_ENABLED' => 'false',}}
|
||||
alter 'index_time_troubleshooting_file_bucket',{METADATA => {'REGION_MEMSTORE_REPLICATION' => 'false', 'hbase.region.replica.wait.for.primary.flush' => 'false','SPLIT_ENABLED' => 'false',}}
|
||||
alter 'index_filename_troubleshooting_file_bucket',{METADATA => {'REGION_MEMSTORE_REPLICATION' => 'false', 'hbase.region.replica.wait.for.primary.flush' => 'false','SPLIT_ENABLED' => 'false',}}
|
||||
alter 'index_partfile_troubleshooting_file_bucket',{METADATA => {'REGION_MEMSTORE_REPLICATION' => 'false', 'hbase.region.replica.wait.for.primary.flush' => 'false','SPLIT_ENABLED' => 'false',}}
|
||||
|
||||
alter 'knowledge_base_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_time_knowledge_base_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_filename_knowledge_base_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_partfile_knowledge_base_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
|
||||
alter 'report_snapshot_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_time_report_snapshot_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_filename_report_snapshot_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_partfile_report_snapshot_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
|
||||
alter 'assessment_file_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_time_assessment_file_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_filename_assessment_file_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
alter 'index_partfile_assessment_file_bucket',{DURABILITY =>'SYNC_WAL'}
|
||||
5
tsg_olap/upgrade/TSG-24.07/hos/create-bucket-24.07.txt
Normal file
5
tsg_olap/upgrade/TSG-24.07/hos/create-bucket-24.07.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
curl -X PUT http://192.168.44.67:9098/hos/traffc_file_bucket -H 'token:c21f969b5f03d33d43e04f8f136e7682' -H 'x-hos-region-count:64*hbase服务器数'
|
||||
curl -X PUT http://192.168.44.67:9098/hos/troubleshooting_file_bucket -H 'token:c21f969b5f03d33d43e04f8f136e7682' -H 'x-hos-region-count:16'
|
||||
curl -X PUT http://192.168.44.67:9098/hos/knowledge_base_bucket -H 'token:c21f969b5f03d33d43e04f8f136e7682' -H 'x-hos-region-count:16' -H 'x-hos-wal:open'
|
||||
curl -X PUT http://192.168.44.67:9098/hos/report_snapshot_bucket -H 'token:c21f969b5f03d33d43e04f8f136e7682' -H 'x-hos-region-count:16' -H 'x-hos-wal:open'
|
||||
curl -X PUT http://192.168.44.67:9098/hos/assessment_file_bucket -H 'token:c21f969b5f03d33d43e04f8f136e7682' -H 'x-hos-region-count:16' -H 'x-hos-wal:open'
|
||||
104
tsg_olap/upgrade/TSG-24.07/hos/galaxy-hos-service.yml
Normal file
104
tsg_olap/upgrade/TSG-24.07/hos/galaxy-hos-service.yml
Normal file
@@ -0,0 +1,104 @@
|
||||
#服务端口
|
||||
server:
|
||||
port: 8186
|
||||
max-http-header-size: 20MB
|
||||
tomcat:
|
||||
max-threads: 400
|
||||
#tomcat缓存大小，单位KB系统默认10M，配置10g
|
||||
tomcat:
|
||||
cacheMaxSize: 1000000
|
||||
#hbase参数
|
||||
hbase:
|
||||
zookeeperQuorum: 192.168.44.11:2181,192.168.44.14:2181,192.168.44.15:2181
|
||||
zookeeperPort: 2181
|
||||
zookeeperNodeParent: /hbase
|
||||
clientRetriesNumber: 9
|
||||
rpcTimeout: 100000
|
||||
connectPool: 10
|
||||
clientWriteBuffer: 10485760
|
||||
clientKeyValueMaxsize: 1073741824
|
||||
mobThreshold: 10485760
|
||||
#part的最大数量
|
||||
maxParts: 100000
|
||||
#每次获取的part数
|
||||
getPartBatch: 10
|
||||
#hbase索引表前缀，前缀为以下的都为索引表
|
||||
timeIndexTablePrefix: index_time_
|
||||
filenameIndexTablePrefix: index_filename_
|
||||
partFileIndexTablePrefix: index_partfile_
|
||||
systemBucketMeta: system:bucket_meta
|
||||
#创建表的分区数
|
||||
regionCount: 16
|
||||
filenameHead: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f
|
||||
partHead: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f
|
||||
#获取文件大小的目录
|
||||
dataPath: /hbase
|
||||
#hadoop集群namenode节点，单机为单个ip，集群为ip1,ip2
|
||||
hadoopNameNodes: 192.168.44.10,192.168.44.11
|
||||
#副本数，单机为1，集群为2
|
||||
hadoopReplication: 2
|
||||
#hadoop端口
|
||||
hadoopPort: 9000
|
||||
hadoopUser: root
|
||||
hadoopNameServices: ns1
|
||||
hadoopNameNodesNs1: nn1,nn2
|
||||
#ttl相关参数
|
||||
ttlScanBatch: 1000
|
||||
ttlDeleteBatch: 1000
|
||||
asyncPut: 0
|
||||
#是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务
|
||||
auth:
|
||||
open: 0
|
||||
#http访问使用的token
|
||||
token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo)
|
||||
#s3验证
|
||||
s3:
|
||||
accesskey: ENC(FUQDvVP+zqCiwHQhXcRvbw==)
|
||||
secretkey: ENC(FUQDvVP+zqCiwHQhXcRvbw==)
|
||||
hos:
|
||||
#文件大小阈值
|
||||
maxFileSize: 5073741800
|
||||
#大文件阈值
|
||||
uploadThreshold: 104857600
|
||||
#长连接超时时间
|
||||
keepAliveTimeout: 60000
|
||||
#批量删除对象的最大数量
|
||||
deleteMultipleNumber: 1000
|
||||
#获取对象列表等操作的最大值
|
||||
maxResultLimit: 100000
|
||||
#分块上传的最大分块数
|
||||
maxPartNumber: 10000
|
||||
#追加上传的最大次数
|
||||
maxAppendNumber: 100000
|
||||
#是否快速上传
|
||||
isQuickUpload: 0
|
||||
#是否快速下载文件，1打开，hbase内存小于20G的集群设为0
|
||||
isQuickDownloadFile: 0
|
||||
#用户白名单（hbase的namespace），获取存储配额
|
||||
users: default
|
||||
#是否打开限流,0:关闭，1:打开
|
||||
openRateLimiter: 0
|
||||
#限流每秒请求数
|
||||
rateLimiterQps: 20000
|
||||
#ttl方式，默认为0
|
||||
ttlMode: 0
|
||||
#执行ttl的线程数
|
||||
ttlThread: 10
|
||||
#设置上传文件大小的最大值
|
||||
spring:
|
||||
servlet:
|
||||
multipart:
|
||||
max-file-size: 5GB
|
||||
max-request-size: 5GB
|
||||
#Prometheus参数
|
||||
application:
|
||||
name: HosServiceApplication
|
||||
#Prometheus参数
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: '*'
|
||||
metrics:
|
||||
tags:
|
||||
application: ${spring.application.name}
|
||||
BIN
tsg_olap/upgrade/TSG-24.07/hos/hosutil/galaxy-hos-util-1.2.jar
Normal file
BIN
tsg_olap/upgrade/TSG-24.07/hos/hosutil/galaxy-hos-util-1.2.jar
Normal file
Binary file not shown.
152
tsg_olap/upgrade/TSG-24.07/hos/hosutil/hosutil.sh
Normal file
152
tsg_olap/upgrade/TSG-24.07/hos/hosutil/hosutil.sh
Normal file
@@ -0,0 +1,152 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo "Usage: ./hosutil.sh [command] [-h] [options...]"
|
||||
echo "Available commands are:"
|
||||
echo " download Download individual or batch files"
|
||||
echo " upload Upload individual or batch files"
|
||||
echo " check Check file availability"
|
||||
echo " combiner Verify if the file-chunk-combiner data stream is correct"
|
||||
echo " version Print the version"
|
||||
echo ""
|
||||
echo "download options:"
|
||||
echo " -b --bucket 要访问的桶。"
|
||||
echo " -d --directory 保存文件的路径，该路径不存在，将被创建，默认为./download/。"
|
||||
echo " -k --keys 要下载的文件名，可以为单个文件或多个文件，多个文件以逗号隔开。"
|
||||
echo " -p --prefix 根据文件名前缀批量下载文件时的文件名前缀。"
|
||||
echo " -s --startTime 起始时间。时间是UTC时间，格式为yyyyMMdd、yyyy-MM-dd、yyyyMMddHHmmss，默认是前一天的时间。"
|
||||
echo " -e --endTime 结束时间。时间是UTC时间，格式为yyyyMMdd、yyyy-MM-dd、yyyyMMddHHmmss，默认是当前时间。"
|
||||
echo " -c --count 设置下载文件的数量，默认为1000，最大值为100000。"
|
||||
echo " -t --threads 线程数，默认为1，最大值为10。"
|
||||
echo "upload options:"
|
||||
echo " -b --bucket 要访问的桶。"
|
||||
echo " -d --directory 上传文件所在的路径，默认为./upload/。"
|
||||
echo " -t --threads 线程数，默认为1，最大值为10。"
|
||||
echo "check options:"
|
||||
echo " -s --startTime 起始时间。时间是UTC时间，格式为yyyyMMdd、yyyy-MM-dd、yyyyMMddHHmmss，默认是前一天的时间。"
|
||||
echo " -e --endTime 结束时间。时间是UTC时间，格式为yyyyMMdd、yyyy-MM-dd、yyyyMMddHHmmss，默认是当前时间。"
|
||||
echo " -c --count 评估的日志数量，默认为1000，最大值为100000。"
|
||||
echo " -l --logType 评估指定日志的文件，不指定该参数则评估所有日志，支持评估多种日志，使用逗号隔开，例如session_record,security_event。支持的日志有security_event、proxy_event、session_record、voip_record、assessment_event、transaction_record、troubleshooting。"
|
||||
echo " -f --fileType 指定文件的类型，不指定该参数则为所有类型，例如pcap。目前类型有：mail、http、pcap、other。目前只有session_record（mail、http、pcap）、security_event（mail、http、pcap）、transaction_record（mail、http）包含多种类型，其他日志省略该参数即可。"
|
||||
echo " -t --threads 线程数，默认为1，最大值为10。"
|
||||
echo "combiner options:"
|
||||
echo " -j --job 要验证的任务，有traffic、troubleshooting、all，all为都验证，默认为all。"
|
||||
echo " -p --prefix 文件名前缀，防止多次执行文件名重复，不可省略。"
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
version="1.2"
|
||||
operation=$1
|
||||
bucket=""
|
||||
directory=""
|
||||
keys=""
|
||||
prefix=""
|
||||
startTime=""
|
||||
endTime=""
|
||||
count=1000
|
||||
threads=1
|
||||
logType=""
|
||||
fileType=""
|
||||
jobName="all"
|
||||
jar="galaxy-hos-util-"$version".jar"
|
||||
|
||||
shift
|
||||
while getopts ":h:b:d:k:p:s:e:c:t:l:f:j:" opt; do
|
||||
case $opt in
|
||||
h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
b)
|
||||
bucket=$OPTARG
|
||||
;;
|
||||
d)
|
||||
directory=$OPTARG
|
||||
;;
|
||||
k)
|
||||
keys=$OPTARG
|
||||
;;
|
||||
p)
|
||||
prefix=$OPTARG
|
||||
;;
|
||||
s)
|
||||
startTime=$OPTARG
|
||||
;;
|
||||
e)
|
||||
endTime=$OPTARG
|
||||
;;
|
||||
c)
|
||||
count=$OPTARG
|
||||
;;
|
||||
t)
|
||||
threads=$OPTARG
|
||||
;;
|
||||
l)
|
||||
logType=$OPTARG
|
||||
;;
|
||||
f)
|
||||
fileType=$OPTARG
|
||||
;;
|
||||
j)
|
||||
jobName=$OPTARG
|
||||
;;
|
||||
\?)
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
:)
|
||||
echo "Option -$OPTARG requires an argument" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
download() {
|
||||
if [ -z "$directory" ]; then
|
||||
directory="./download/"
|
||||
fi
|
||||
java -jar $jar download $bucket $directory keys=$keys prefix=$prefix maxKeys=$count timeRange=$startTime/$endTime threadNum=$threads
|
||||
}
|
||||
|
||||
upload() {
|
||||
if [ -z "$directory" ]; then
|
||||
directory="./upload/"
|
||||
fi
|
||||
java -jar $jar upload $bucket $directory threadNum=$threads
|
||||
}
|
||||
|
||||
check() {
|
||||
java -jar $jar check logType=$logType fileType=$fileType maxLogs=$count timeRange=$startTime/$endTime threadNum=$threads
|
||||
}
|
||||
|
||||
combiner() {
|
||||
java -jar $jar combiner $jobName $prefix
|
||||
}
|
||||
|
||||
if [ "$operation" = "download" ];then
|
||||
download
|
||||
exit 0
|
||||
elif [ "$operation" = "upload" ];then
|
||||
upload
|
||||
exit 0
|
||||
elif [ "$operation" = "check" ];then
|
||||
check
|
||||
exit 0
|
||||
elif [ "$operation" = "combiner" ];then
|
||||
combiner
|
||||
exit 0
|
||||
elif [ "$operation" = "version" ];then
|
||||
echo $version
|
||||
exit 0
|
||||
else
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
Reference in New Issue
Block a user