Merge remote-tracking branch 'origin/master'
This commit is contained in:
zhanghongqing
22
tsg_olap/upgrade/TSG-24.07/hos/hosutil/config.properties
Normal file
22
tsg_olap/upgrade/TSG-24.07/hos/hosutil/config.properties
Normal file
@@ -0,0 +1,22 @@
|
||||
qgw.serverAddr=http://192.168.44.67:9999
|
||||
hos.serverAddr=http://192.168.44.67:9098
|
||||
hos.token=c21f969b5f03d33d43e04f8f136e7682
|
||||
kafka.server=192.168.44.11:9092
|
||||
|
||||
#延迟时间，校验多少秒之前的文件，单位秒
|
||||
hos.timeDelay=180
|
||||
kafka.traffic.topic=TRAFFIC-FILE-STREAM-RECORD
|
||||
kafka.troubleshooting.topic=TROUBLESHOOTING-FILE-STREAM-RECORD
|
||||
file.chunk.combiner.window.time=11000
|
||||
traffic.file.count=0
|
||||
threads=1
|
||||
maxThreads=10
|
||||
printOutInterval=1000
|
||||
http.maxTotal=100
|
||||
http.defaultMaxPerRoute=100
|
||||
http.connectTimeout=5000
|
||||
http.connectionRequestTimeout=10000
|
||||
http.socketTimeout=-1
|
||||
http.staleConnectionCheckEnabled=true
|
||||
hos.logTypes=security_event,monitor_event,proxy_event,session_record,voip_record,assessment_event,transaction_record,troubleshooting
|
||||
hos.logTypesFileTypesFileUrlFields=security_event:http-http_response_body&http_request_body,pacp-packet_capture_file,eml-mail_eml_file;proxy_event:http-http_response_body&http_request_body;session_record:http-http_response_body&http_request_body,pacp-packet_capture_file,eml-mail_eml_file;voip_record:pacp-rtp_pcap_path;assessment_event:other-assessment_file;transaction_record:http-http_response_body&http_request_body,eml-mail_eml_file;monitor_event:http-http_response_body&http_request_body,pacp-packet_capture_file,eml-mail_eml_file
|
||||
Binary file not shown.
@@ -26,12 +26,12 @@ usage() {
|
||||
echo " -s --startTime 起始时间。时间是UTC时间，格式为yyyyMMdd、yyyy-MM-dd、yyyyMMddHHmmss，默认是前一天的时间。"
|
||||
echo " -e --endTime 结束时间。时间是UTC时间，格式为yyyyMMdd、yyyy-MM-dd、yyyyMMddHHmmss，默认是当前时间。"
|
||||
echo " -c --count 评估的日志数量，默认为1000，最大值为100000。"
|
||||
echo " -l --logType 评估指定日志的文件，不指定该参数则评估所有日志，支持评估多种日志，使用逗号隔开，例如session_record,security_event。支持的日志有security_event、proxy_event、session_record、voip_record、assessment_event、transaction_record、troubleshooting。"
|
||||
echo " -f --fileType 指定文件的类型，不指定该参数则为所有类型，例如pcap。目前类型有：mail、http、pcap、other。目前只有session_record（mail、http、pcap）、security_event（mail、http、pcap）、transaction_record（mail、http）包含多种类型，其他日志省略该参数即可。"
|
||||
echo " -d --dataCenter 评估的数据中心，可指定多个，使用逗号隔开，不指定为所有数据中心。只有security_event、monitor_event、proxy_event、session_record、voip_record支持选择数据中心"
|
||||
echo " -l --logType 评估指定日志的文件，不指定该参数则评估所有日志，支持评估多种日志，使用逗号隔开，例如session_record,security_event。支持的日志有security_event、monitor_event、proxy_event、session_record、voip_record、assessment_event、transaction_record、troubleshooting。"
|
||||
echo " -f --fileType 指定文件的类型，不指定该参数则为所有类型，例如pcap。目前类型有：mail、http、pcap、other。目前只有session_record（mail、http、pcap）、security_event（mail、http、pcap）、monitor_event（mail、http、pcap）、transaction_record（mail、http）包含多种类型，其他日志省略该参数即可。"
|
||||
echo " -t --threads 线程数，默认为1，最大值为10。"
|
||||
echo "combiner options:"
|
||||
echo " -j --job 要验证的任务，有traffic、troubleshooting、all，all为都验证，默认为all。"
|
||||
echo " -p --prefix 文件名前缀，防止多次执行文件名重复，不可省略。"
|
||||
echo " -j --job 要验证的任务，有traffic、troubleshooting，都验证指定为traffic,troubleshooting，不指定该参数默认为traffic。（24.05已删除troubleshooting任务）"
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
@@ -39,7 +39,7 @@ if [ $# -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
version="1.2"
|
||||
version="1.3"
|
||||
operation=$1
|
||||
bucket=""
|
||||
directory=""
|
||||
@@ -51,7 +51,8 @@ count=1000
|
||||
threads=1
|
||||
logType=""
|
||||
fileType=""
|
||||
jobName="all"
|
||||
dataCenter=""
|
||||
jobName="traffic"
|
||||
jar="galaxy-hos-util-"$version".jar"
|
||||
|
||||
shift
|
||||
@@ -65,7 +66,11 @@ while getopts ":h:b:d:k:p:s:e:c:t:l:f:j:" opt; do
|
||||
bucket=$OPTARG
|
||||
;;
|
||||
d)
|
||||
directory=$OPTARG
|
||||
if [ "$operation" == "check" ]; then
|
||||
dataCenter=$OPTARG
|
||||
else
|
||||
directory=$OPTARG
|
||||
fi
|
||||
;;
|
||||
k)
|
||||
keys=$OPTARG
|
||||
@@ -122,11 +127,11 @@ java -jar $jar upload $bucket $directory threadNum=$threads
|
||||
}
|
||||
|
||||
check() {
|
||||
java -jar $jar check logType=$logType fileType=$fileType maxLogs=$count timeRange=$startTime/$endTime threadNum=$threads
|
||||
java -jar $jar check dataCenter=$dataCenter logType=$logType fileType=$fileType maxLogs=$count timeRange=$startTime/$endTime threadNum=$threads
|
||||
}
|
||||
|
||||
combiner() {
|
||||
java -jar $jar combiner $jobName $prefix
|
||||
java -jar $jar combiner $jobName
|
||||
}
|
||||
|
||||
if [ "$operation" = "download" ];then
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,22 @@
|
||||
SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha
|
||||
FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT vsys_id, recv_time, log_id, profile_id, rule_id, start_time, end_time, attack_type, severity, conditions, destination_ip, destination_country, source_ip_list, source_country_list, sessions, session_rate, packets, packet_rate, bytes, bit_rate
|
||||
FROM tsg_galaxy_v3.dos_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.monitor_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, shaping_rule_list, proxy_rule_list, statistics_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, doh_url, doh_host, doh_request_line, doh_response_line, doh_cookie, doh_referer, doh_user_agent, doh_content_length, doh_content_type, doh_set_cookie, doh_version, doh_message_id, doh_qr, doh_opcode, doh_aa, doh_tc, doh_rd, doh_ra, doh_rcode, doh_qdcount, doh_ancount, doh_nscount, doh_arcount, doh_qname, doh_qtype, doh_qclass, doh_cname, doh_sub, doh_rr, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.proxy_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.security_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, c2s_ttl, s2c_ttl, security_rule_list, security_action, monitor_rule_list, sc_rule_list, sc_rsp_raw, sc_rsp_decrypted, shaping_rule_list, proxy_rule_list, statistics_rule_list, proxy_action, proxy_pinning_status, proxy_intercept_status, proxy_passthrough_reason, proxy_client_side_latency_ms, proxy_server_side_latency_ms, proxy_client_side_version, proxy_server_side_version, proxy_cert_verify, proxy_intercept_error, monitor_mirrored_pkts, monitor_mirrored_bytes, client_ip, client_ip_tags, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, subscriber_id, imei, imsi, phone_number, apn, server_ip, server_ip_tags, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, server_fqdn, server_fqdn_tags, server_domain, app_transition, app, app_category, app_debug_info, app_content, app_extra_info, fqdn_category_list, ip_protocol, decoded_path, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_handshake_latency_ms, ssl_ja3_hash, ssl_ja3s_hash, ssl_cert_issuer, ssl_cert_subject, ssl_esni_flag, ssl_ech_flag, dtls_cookie, dtls_version, dtls_sni, dtls_san, dtls_cn, dtls_handshake_latency_ms, dtls_ja3_fingerprint, dtls_ja3_hash, dtls_cert_issuer, dtls_cert_subject, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, ftp_account, ftp_url, ftp_link_type, quic_version, quic_sni, quic_user_agent, rdp_cookie, rdp_security_protocol, rdp_client_channels, rdp_keyboard_layout, rdp_client_version, rdp_client_name, rdp_client_product_id, rdp_desktop_width, rdp_desktop_height, rdp_requested_color_depth, rdp_certificate_type, rdp_certificate_count, rdp_certificate_permanent, rdp_encryption_level, rdp_encryption_method, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, stratum_mining_subscribe, sent_pkts, received_pkts, sent_bytes, received_bytes, tcp_c2s_ip_fragments, tcp_s2c_ip_fragments, tcp_c2s_lost_bytes, tcp_s2c_lost_bytes, tcp_c2s_o3_pkts, tcp_s2c_o3_pkts, tcp_c2s_rtx_pkts, tcp_s2c_rtx_pkts, tcp_c2s_rtx_bytes, tcp_s2c_rtx_bytes, tcp_rtt_ms, tcp_client_isn, tcp_server_isn, packet_capture_file, in_src_mac, out_src_mac, in_dest_mac, out_dest_mac, encapsulation, dup_traffic_flag, tunnel_id_list, tunnel_endpoint_a_desc, tunnel_endpoint_b_desc
|
||||
FROM tsg_galaxy_v3.session_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, ingestion_time, processing_time, insert_time, address_type, vsys_id, client_ip, client_port, server_ip, server_port, sent_pkts, received_pkts, sent_bytes, received_bytes, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, http_url, http_host, http_request_line, http_response_line, http_request_body, http_response_body, http_proxy_flag, http_sequence, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_set_cookie, http_version, http_status_code, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_from_cmd, mail_to_cmd, mail_from, mail_password, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_attachment_name, mail_attachment_name_charset, mail_starttls_flag, mail_eml_file, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye
|
||||
FROM tsg_galaxy_v3.transaction_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT recv_time, log_id, decoded_as, session_id, start_timestamp_ms, end_timestamp_ms, duration_ms, tcp_handshake_latency_ms, ingestion_time, processing_time, insert_time, device_id, out_link_id, in_link_id, device_tag, data_center, device_group, sled_ip, address_type, direction, vsys_id, t_vsys_id, flags, flags_identify_info, client_ip, client_port, client_os_desc, client_geolocation, client_country, client_super_administrative_area, client_administrative_area, client_sub_administrative_area, client_asn, server_ip, server_port, server_os_desc, server_geolocation, server_country, server_super_administrative_area, server_administrative_area, server_sub_administrative_area, server_asn, ip_protocol, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, sent_pkts, received_pkts, sent_bytes, received_bytes
|
||||
FROM tsg_galaxy_v3.voip_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT log_id, recv_time, vsys_id, timestamp_us, egress_action, job_id, sled_ip, device_group, traffic_link_id, source_ip, source_port, destination_ip, destination_port, packet, packet_length, measurements
|
||||
FROM tsg_galaxy_v3.datapath_telemetry_record where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
SELECT log_id, recv_time, vsys_id, device_id, device_group, data_center, direction, ip_protocol, client_ip, server_ip, internal_ip, external_ip, client_country, server_country, client_asn, server_asn, server_fqdn, server_domain, app, app_category, c2s_ttl, s2c_ttl, c2s_link_id, s2c_link_id, sessions, bytes, sent_bytes, received_bytes, pkts, sent_pkts, received_pkts, asymmetric_c2s_flows, asymmetric_s2c_flows, c2s_fragments, s2c_fragments, c2s_tcp_lost_bytes, s2c_tcp_lost_bytes, c2s_tcp_retransmitted_pkts, s2c_tcp_retransmitted_pkts
|
||||
FROM tsg_galaxy_v3.traffic_sketch_metric where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time <toUnixTimestamp('2030-01-01 00:00:01');
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,143 @@
|
||||
sources:
|
||||
kafka_source:
|
||||
type: kafka
|
||||
properties:
|
||||
topic: TRAFFIC-SKETCH-METRIC
|
||||
kafka.bootstrap.servers: "{{ kafka_source_servers }}"
|
||||
kafka.session.timeout.ms: 60000
|
||||
kafka.max.poll.records: 3000
|
||||
kafka.max.partition.fetch.bytes: 31457280
|
||||
kafka.security.protocol: SASL_PLAINTEXT
|
||||
kafka.sasl.mechanism: PLAIN
|
||||
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
|
||||
kafka.group.id: etl_traffic_sketch_metric
|
||||
kafka.auto.offset.reset: latest
|
||||
kafka.compression.type: none
|
||||
format: json
|
||||
|
||||
processing_pipelines:
|
||||
pre_etl_processor: # [object] Processing Pipeline
|
||||
type: projection
|
||||
remove_fields:
|
||||
output_fields:
|
||||
functions: # [array of object] Function List
|
||||
- function: UNIX_TIMESTAMP_CONVERTER
|
||||
lookup_fields: [ timestamp_ms ]
|
||||
output_fields: [ recv_time ]
|
||||
parameters:
|
||||
precision: seconds
|
||||
interval: 300
|
||||
|
||||
aggregate_processor:
|
||||
type: aggregate
|
||||
group_by_fields: [vsys_id,device_id,device_group,data_center,ip_protocol,direction,client_ip,server_ip,server_domain,app,recv_time]
|
||||
window_type: tumbling_processing_time # tumbling_event_time,sliding_processing_time,sliding_event_time
|
||||
window_size: 300
|
||||
functions:
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ sessions ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ sent_bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ received_bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ pkts ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ sent_pkts ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ received_pkts ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ asymmetric_c2s_flows ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ asymmetric_s2c_flows ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ c2s_fragments ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ s2c_fragments ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ c2s_tcp_lost_bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ s2c_tcp_lost_bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ c2s_tcp_retransmitted_pkts ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ s2c_tcp_retransmitted_pkts ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ client_country ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ server_country ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ client_asn ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ server_asn ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ server_fqdn ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ app_category ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ c2s_ttl ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ s2c_ttl ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ c2s_link_id ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ s2c_link_id ]
|
||||
|
||||
|
||||
|
||||
post_etl_processor: # [object] Processing Pipeline
|
||||
type: projection
|
||||
remove_fields:
|
||||
output_fields:
|
||||
functions: # [array of object] Function List
|
||||
- function: EVAL
|
||||
output_fields: [ internal_ip ]
|
||||
parameters:
|
||||
value_expression: 'direction=Outbound? client_ip : server_ip'
|
||||
- function: EVAL
|
||||
output_fields: [ external_ip ]
|
||||
parameters:
|
||||
value_expression: 'direction=Outbound? server_ip : client_ip'
|
||||
|
||||
- function: SNOWFLAKE_ID
|
||||
lookup_fields: [ '' ]
|
||||
output_fields: [ log_id ]
|
||||
filter:
|
||||
parameters:
|
||||
data_center_id_num: 1
|
||||
|
||||
|
||||
sinks:
|
||||
clickhouse_sink:
|
||||
type: clickhouse
|
||||
properties:
|
||||
host: "{{ clickhouse_servers }}"
|
||||
table: tsg_galaxy_v3.traffic_sketch_metric_local
|
||||
batch.size: 100000
|
||||
batch.interval: 30s
|
||||
connection.user: e54c9568586180eede1506eecf3574e9
|
||||
connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
|
||||
|
||||
|
||||
application:
|
||||
|
||||
env: # [object] Environment Variables
|
||||
name: etl_traffic_sketch_metric # [string] Job Name
|
||||
shade.identifier: aes
|
||||
pipeline:
|
||||
object-reuse: true # [boolean] Object Reuse, default is false
|
||||
topology:
|
||||
topology:
|
||||
- name: kafka_source
|
||||
downstream: [pre_etl_processor]
|
||||
- name: pre_etl_processor
|
||||
downstream: [aggregate_processor]
|
||||
- name: aggregate_processor
|
||||
downstream: [post_etl_processor]
|
||||
- name: post_etl_processor
|
||||
downstream: [clickhouse_sink]
|
||||
- name: clickhouse_sink
|
||||
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
sources:
|
||||
kafka_source:
|
||||
type: kafka
|
||||
properties:
|
||||
topic: TRAFFIC-SKETCH-METRIC
|
||||
kafka.bootstrap.servers: {{ kafka_source_servers }}
|
||||
kafka.session.timeout.ms: 60000
|
||||
kafka.max.poll.records: 3000
|
||||
kafka.max.partition.fetch.bytes: 31457280
|
||||
kafka.security.protocol: SASL_PLAINTEXT
|
||||
kafka.sasl.mechanism: PLAIN
|
||||
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
|
||||
kafka.group.id: etl_traffic_sketch_metric
|
||||
kafka.auto.offset.reset: latest
|
||||
kafka.compression.type: none
|
||||
format: json
|
||||
|
||||
processing_pipelines:
|
||||
pre_etl_processor: # [object] Processing Pipeline
|
||||
type: projection
|
||||
remove_fields:
|
||||
output_fields:
|
||||
functions: # [array of object] Function List
|
||||
- function: UNIX_TIMESTAMP_CONVERTER
|
||||
lookup_fields: [ timestamp_ms ]
|
||||
output_fields: [ recv_time ]
|
||||
parameters:
|
||||
precision: seconds
|
||||
interval: 300
|
||||
|
||||
aggregate_processor:
|
||||
type: aggregate
|
||||
group_by_fields: [vsys_id,device_id,device_group,data_center,ip_protocol,direction,client_ip,server_ip,server_domain,app,recv_time]
|
||||
window_type: tumbling_processing_time # tumbling_event_time,sliding_processing_time,sliding_event_time
|
||||
window_size: 300
|
||||
functions:
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ sessions ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ sent_bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ received_bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ pkts ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ sent_pkts ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ received_pkts ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ asymmetric_c2s_flows ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ asymmetric_s2c_flows ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ c2s_fragments ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ s2c_fragments ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ c2s_tcp_lost_bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ s2c_tcp_lost_bytes ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ c2s_tcp_retransmitted_pkts ]
|
||||
- function: NUMBER_SUM
|
||||
lookup_fields: [ s2c_tcp_retransmitted_pkts ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ client_country ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ server_country ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ client_asn ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ server_asn ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ server_fqdn ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ app_category ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ c2s_ttl ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ s2c_ttl ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ c2s_link_id ]
|
||||
- function: FIRST_VALUE
|
||||
lookup_fields: [ s2c_link_id ]
|
||||
|
||||
|
||||
|
||||
post_etl_processor: # [object] Processing Pipeline
|
||||
type: projection
|
||||
remove_fields:
|
||||
output_fields:
|
||||
functions: # [array of object] Function List
|
||||
- function: EVAL
|
||||
output_fields: [ internal_ip ]
|
||||
parameters:
|
||||
value_expression: 'direction=Outbound? client_ip : server_ip'
|
||||
- function: EVAL
|
||||
output_fields: [ external_ip ]
|
||||
parameters:
|
||||
value_expression: 'direction=Outbound? server_ip : client_ip'
|
||||
|
||||
- function: SNOWFLAKE_ID
|
||||
lookup_fields: [ '' ]
|
||||
output_fields: [ log_id ]
|
||||
filter:
|
||||
parameters:
|
||||
data_center_id_num: 1
|
||||
|
||||
|
||||
sinks:
|
||||
kafka_sink:
|
||||
type: kafka
|
||||
properties:
|
||||
topic: TRAFFIC-SKETCH-METRIC
|
||||
kafka.bootstrap.servers: {{ kafka_sink_servers }}
|
||||
kafka.retries: 0
|
||||
kafka.linger.ms: 10
|
||||
kafka.request.timeout.ms: 30000
|
||||
kafka.batch.size: 262144
|
||||
kafka.buffer.memory: 134217728
|
||||
kafka.max.request.size: 10485760
|
||||
kafka.compression.type: snappy
|
||||
kafka.security.protocol: SASL_PLAINTEXT
|
||||
kafka.sasl.mechanism: PLAIN
|
||||
kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817
|
||||
format: json
|
||||
json.ignore.parse.errors: false
|
||||
log.failures.only: true
|
||||
|
||||
clickhouse_sink:
|
||||
type: clickhouse
|
||||
properties:
|
||||
host: {{ clickhouse_servers }}
|
||||
table: tsg_galaxy_v3.traffic_sketch_metric_local
|
||||
batch.size: 100000
|
||||
batch.interval: 30s
|
||||
connection.user: e54c9568586180eede1506eecf3574e9
|
||||
connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e
|
||||
|
||||
|
||||
application:
|
||||
|
||||
env: # [object] Environment Variables
|
||||
name: etl_traffic_sketch_metric # [string] Job Name
|
||||
shade.identifier: aes
|
||||
pipeline:
|
||||
object-reuse: true # [boolean] Object Reuse, default is false
|
||||
{{ topology }}
|
||||
|
||||
|
||||
24
tsg_olap/upgrade/TSG-24.08/groot_stream/udf.plugins
Normal file
24
tsg_olap/upgrade/TSG-24.08/groot_stream/udf.plugins
Normal file
@@ -0,0 +1,24 @@
|
||||
com.geedgenetworks.core.udf.AsnLookup
|
||||
com.geedgenetworks.core.udf.CurrentUnixTimestamp
|
||||
com.geedgenetworks.core.udf.DecodeBase64
|
||||
com.geedgenetworks.core.udf.Domain
|
||||
com.geedgenetworks.core.udf.Drop
|
||||
com.geedgenetworks.core.udf.EncodeBase64
|
||||
com.geedgenetworks.core.udf.Eval
|
||||
com.geedgenetworks.core.udf.Flatten
|
||||
com.geedgenetworks.core.udf.FromUnixTimestamp
|
||||
com.geedgenetworks.core.udf.GenerateStringArray
|
||||
com.geedgenetworks.core.udf.GeoIpLookup
|
||||
com.geedgenetworks.core.udf.JsonExtract
|
||||
com.geedgenetworks.core.udf.PathCombine
|
||||
com.geedgenetworks.core.udf.Rename
|
||||
com.geedgenetworks.core.udf.SnowflakeId
|
||||
com.geedgenetworks.core.udf.StringJoiner
|
||||
com.geedgenetworks.core.udf.UnixTimestampConverter
|
||||
com.geedgenetworks.core.udf.udaf.NumberSum
|
||||
com.geedgenetworks.core.udf.udaf.CollectList
|
||||
com.geedgenetworks.core.udf.udaf.CollectSet
|
||||
com.geedgenetworks.core.udf.udaf.LongCount
|
||||
com.geedgenetworks.core.udf.udaf.Mean
|
||||
com.geedgenetworks.core.udf.udaf.LastValue
|
||||
com.geedgenetworks.core.udf.udaf.FirstValue
|
||||
Reference in New Issue
Block a user