diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 199759a..99512f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ variables: BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux" BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/ INSTALL_PREFIX: "/home/mesasoft/sapp_run/" - INSTALL_DEPENDENCY_LIBRARY: systemd-devel libbreakpad_mini numactl-devel zlib-devel vim-common libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaat4-4.0.31.73887d5 libmaat4-devel-4.0.31.73887d5 quic-devel mesa_sip-devel gtp-devel libMESA_htable-devel libasan mrzcpd rapidjson-devel libMESA_jump_layer-devel stratum-devel rdp-devel dtls-devel-1.0.10.ab7166e libfieldstat3-devel openssl-devel + INSTALL_DEPENDENCY_LIBRARY: systemd-devel libbreakpad_mini numactl-devel zlib-devel vim-common libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaat4-4.0.31.73887d5 libmaat4-devel-4.0.31.73887d5 quic-devel mesa_sip-devel gtp-devel libMESA_htable-devel libasan mrzcpd rapidjson-devel libMESA_jump_layer-devel stratum-devel rdp-devel dtls-devel-1.0.10.ab7166e libfieldstat3-devel openssl-devel libipfix_exporter stages: - analysis diff --git a/CMakeLists.txt b/CMakeLists.txt index 9120e26..80b56f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,7 @@ install(FILES bin/tsg_log_field.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf install(FILES bin/app_l7_proto_id.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) install(FILES bin/tsg_l7_protocol.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) install(FILES bin/tsg_maat.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) +install(FILES bin/ipfix_conf.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) install(FILES bin/HTTP200.html DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) install(FILES bin/HTTP204.html DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) diff --git a/bin/ipfix_conf.json b/bin/ipfix_conf.json new file mode 100644 index 0000000..ba00e2d --- /dev/null +++ b/bin/ipfix_conf.json @@ -0,0 +1,1143 @@ +{ + "version":10, + "collector_ip": "192.168.38.214", + "collector_port": 4739, + "device_name": "test_device", + "domain_id": 1, + "PEN_number": 54450, + "templates": [ + { + "template_id": 257, + "template_name": "BASE", + "elements":[ + "BASE_elements" + ] + }, + { + "template_id": 258, + "template_name": "SSL", + "elements":[ + "BASE_elements", + "SSL_elements" + ] + }, + { + "template_id": 259, + "template_name": "HTTP", + "elements":[ + "BASE_elements", + "HTTP_elements" + ] + }, + { + "template_id": 260, + "template_name": "MAIL", + "elements":[ + "BASE_elements", + "MAIL_elements" + ] + }, + { + "template_id": 261, + "template_name": "DNS", + "elements":[ + "BASE_elements", + "DNS_elements" + ] + }, + { + "template_id": 262, + "template_name": "DTLS", + "elements":[ + "BASE_elements", + "DTLS_elements" + ] + }, + { + "template_id": 263, + "template_name": "QUIC", + "elements":[ + "BASE_elements", + "QUIC_elements" + ] + }, + { + "template_id": 264, + "template_name": "FTP", + "elements":[ + "BASE_elements", + "FTP_elements" + ] + }, + { + "template_id": 265, + "template_name": "SIP", + "elements":[ + "BASE_elements", + "SIP_elements" + ] + }, + { + "template_id": 266, + "template_name": "RTP", + "elements":[ + "BASE_elements", + "RTP_elements" + ] + }, + { + "template_id": 267, + "template_name": "SSH", + "elements":[ + "BASE_elements", + "SSH_elements" + ] + }, + { + "template_id": 268, + "template_name": "BGP", + "elements":[ + "BASE_elements", + "BGP_elements" + ] + }, + { + "template_id": 269, + "template_name": "RDP", + "elements":[ + "BASE_elements", + "RDP_elements" + ] + }, + { + "template_id": 270, + "template_name": "Stratum", + "elements":[ + "BASE_elements", + "Stratum_elements" + ] + } + ], + "BASE_elements": [ + { + "element_name": "common_stream_trace_id", + "element_type": "string", + "element_id": 1 + }, + { + "element_name": "common_start_time", + "element_type": "unsigned32", + "element_id": 2 + }, + { + "element_name": "common_end_time", + "element_type": "unsigned32", + "element_id": 3 + }, + { + "element_name": "common_con_duration_ms", + "element_type": "unsigned32", + "element_id": 4 + }, + { + "element_name": "common_establish_latency_ms", + "element_type": "unsigned32", + "element_id": 5 + }, + { + "element_name": "common_device_id", + "element_type": "string", + "element_id": 6 + }, + { + "element_name": "common_sled_ip", + "element_type": "string", + "element_id": 7 + }, + { + "element_name": "common_out_link_id", + "element_type": "unsigned32", + "element_id": 8 + }, + { + "element_name": "common_in_link_id", + "element_type": "unsigned32", + "element_id": 9 + }, + { + "element_name": "common_device_tag", + "element_type": "string", + "element_id": 10 + }, + { + "element_name": "common_address_type", + "element_type": "unsigned8", + "element_id": 11 + }, + { + "element_name": "common_schema_type", + "element_type": "string", + "element_id": 12 + }, + { + "element_name": "common_vsys_id", + "element_type": "unsigned32", + "element_id": 13 + }, + { + "element_name": "common_t_vsys_id", + "element_type": "unsigned32", + "element_id": 14 + }, + { + "element_name":"common_flags", + "element_type":"unsigned64", + "element_id": 15 + }, + { + "element_name":"common_flags_identify_info", + "element_type":"string", + "element_id": 16 + }, + { + "element_name":"common_action", + "element_type":"unsigned32", + "element_id": 17 + }, + { + "element_name":"common_shaping_rule_ids", + "element_type":"string", + "element_id": 18 + }, + { + "element_name":"common_client_ip", + "element_type":"string", + "element_id": 19 + }, + { + "element_name":"common_client_port", + "element_type":"unsigned16", + "element_id": 20 + }, + { + "element_name":"common_client_location", + "element_type":"string", + "element_id": 21 + }, + { + "element_name":"common_client_asn", + "element_type":"string", + "element_id": 22 + }, + { + "element_name":"common_subscriber_id", + "element_type":"string", + "element_id": 23 + }, + { + "element_name":"common_imei", + "element_type":"string", + "element_id": 24 + }, + { + "element_name":"common_imsi", + "element_type":"string", + "element_id": 25 + }, + { + "element_name":"common_phone_number", + "element_type":"string", + "element_id": 26 + }, + { + "element_name":"common_in_src_mac", + "element_type":"string", + "element_id": 27 + }, + { + "element_name":"common_out_src_mac", + "element_type":"string", + "element_id": 28 + }, + { + "element_name":"common_server_ip", + "element_type":"string", + "element_id": 29 + }, + { + "element_name":"common_server_port", + "element_type":"unsigned16", + "element_id": 30 + }, + { + "element_name":"common_server_location", + "element_type":"string", + "element_id": 31 + }, + { + "element_name":"common_server_asn", + "element_type":"string", + "element_id": 32 + }, + { + "element_name":"common_in_dest_mac", + "element_type":"string", + "element_id": 33 + }, + { + "element_name":"common_out_dest_mac", + "element_type":"string", + "element_id": 34 + }, + { + "element_name":"common_app_id", + "element_type":"string", + "element_id": 35 + }, + { + "element_name":"common_app_full_path", + "element_type":"string", + "element_id": 36 + }, + { + "element_name":"common_userdefine_app_name", + "element_type":"string", + "element_id": 37 + }, + { + "element_name":"common_app_identify_info", + "element_type":"string", + "element_id": 38 + }, + { + "element_name":"common_app_label", + "element_type":"string", + "element_id": 39 + }, + { + "element_name":"common_app_behavior", + "element_type":"string", + "element_id": 40 + }, + { + "element_name":"common_app_surrogate_id", + "element_type":"string", + "element_id": 41 + }, + { + "element_name":"common_protocol_label", + "element_type":"string", + "element_id": 42 + }, + { + "element_name":"common_service_category", + "element_type":"string", + "element_id": 43 + }, + { + "element_name":"common_l4_protocol", + "element_type":"string", + "element_id": 44 + }, + { + "element_name":"common_sessions", + "element_type":"unsigned32", + "element_id": 45 + }, + { + "element_name":"common_c2s_pkt_num", + "element_type":"unsigned32", + "element_id": 46 + }, + { + "element_name":"common_s2c_pkt_num", + "element_type":"unsigned32", + "element_id": 47 + }, + { + "element_name":"common_c2s_pkt_diff", + "element_type":"unsigned32", + "element_id": 48 + }, + { + "element_name":"common_s2c_pkt_diff", + "element_type":"unsigned32", + "element_id": 49 + }, + { + "element_name":"common_c2s_byte_num", + "element_type":"unsigned64", + "element_id": 50 + }, + { + "element_name":"common_s2c_byte_num", + "element_type":"unsigned64", + "element_id": 51 + }, + { + "element_name":"common_c2s_byte_diff", + "element_type":"unsigned64", + "element_id": 52 + }, + { + "element_name":"common_s2c_byte_diff", + "element_type":"unsigned64", + "element_id": 53 + }, + { + "element_name":"common_c2s_ipfrag_num", + "element_type":"unsigned32", + "element_id": 54 + }, + { + "element_name":"common_s2c_ipfrag_num", + "element_type":"unsigned32", + "element_id": 55 + }, + { + "element_name":"common_c2s_tcp_lostlen", + "element_type":"unsigned64", + "element_id": 56 + }, + { + "element_name":"common_s2c_tcp_lostlen", + "element_type":"unsigned64", + "element_id": 57 + }, + { + "element_name":"common_c2s_tcp_unorder_num", + "element_type":"unsigned32", + "element_id": 58 + }, + { + "element_name":"common_s2c_tcp_unorder_num", + "element_type":"unsigned32", + "element_id": 59 + }, + { + "element_name":"common_c2s_pkt_retrans", + "element_type":"unsigned32", + "element_id": 60 + }, + { + "element_name":"common_s2c_pkt_retrans", + "element_type":"unsigned32", + "element_id": 61 + }, + { + "element_name":"common_c2s_byte_retrans", + "element_type":"unsigned64", + "element_id": 62 + }, + { + "element_name":"common_s2c_byte_retrans", + "element_type":"unsigned64", + "element_id": 63 + }, + { + "element_name":"common_first_ttl", + "element_type":"unsigned32", + "element_id": 64 + }, + { + "element_name":"common_tcp_client_isn", + "element_type":"unsigned32", + "element_id": 65 + }, + { + "element_name":"common_tcp_server_isn", + "element_type":"unsigned32", + "element_id": 66 + }, + { + "element_name":"common_tunnels", + "element_type":"string", + "element_id": 67 + }, + { + "element_name":"common_address_list", + "element_type":"string", + "element_id": 68 + }, + { + "element_name":"common_link_info_c2s", + "element_type":"string", + "element_id": 69 + }, + { + "element_name":"common_link_info_s2c", + "element_type":"string", + "element_id": 70 + }, + { + "element_name":"common_tunnel_endpoint_a_desc", + "element_type":"string", + "element_id": 71 + }, + { + "element_name":"common_tunnel_endpoint_b_desc", + "element_type":"string", + "element_id": 72 + } + ], + "SSL_elements": [ + { + "element_name": "ssl_version", + "element_type": "string", + "element_id": 122 + }, + { + "element_name": "ssl_sni", + "element_type": "string", + "element_id": 123 + }, + { + "element_name": "ssl_san", + "element_type": "string", + "element_id": 124 + }, + { + "element_name": "ssl_cn", + "element_type": "string", + "element_id": 125 + }, + { + "element_name": "ssl_con_latency_ms", + "element_type": "unsigned32", + "element_id": 126 + }, + { + "element_name": "ssl_ja3_fingerprint", + "element_type": "string", + "element_id": 127 + }, + { + "element_name": "ssl_ja3_hash", + "element_type": "string", + "element_id": 128 + }, + { + "element_name": "ssl_ja3s_hash", + "element_type": "string", + "element_id": 129 + }, + { + "element_name": "ssl_cert_issuer", + "element_type": "string", + "element_id": 130 + }, + { + "element_name": "ssl_cert_subject", + "element_type": "string", + "element_id": 131 + }, + { + "element_name": "ssl_ja3s_fingerprint", + "element_type": "string", + "element_id": 132 + } + ], + "HTTP_elements": [ + { + "element_name": "http_url", + "element_type": "string", + "element_id": 74 + }, + { + "element_name": "http_host", + "element_type": "string", + "element_id": 75 + }, + { + "element_name": "http_request_line", + "element_type": "string", + "element_id": 76 + }, + { + "element_name": "http_response_line", + "element_type": "string", + "element_id": 77 + }, + { + "element_name": "http_sequence", + "element_type": "unsigned32", + "element_id": 78 + }, + { + "element_name": "http_cookie", + "element_type": "string", + "element_id": 79 + }, + { + "element_name": "http_referer", + "element_type": "string", + "element_id": 80 + }, + { + "element_name": "http_user_agent", + "element_type": "string", + "element_id": 81 + }, + { + "element_name": "http_request_content_length", + "element_type": "string", + "element_id": 82 + }, + { + "element_name": "http_request_content_type", + "element_type": "string", + "element_id": 83 + }, + { + "element_name": "http_response_content_length", + "element_type": "string", + "element_id": 84 + }, + { + "element_name": "http_response_content_type", + "element_type": "string", + "element_id": 85 + }, + { + "element_name": "http_set_cookie", + "element_type": "string", + "element_id": 86 + }, + { + "element_name": "http_version", + "element_type": "string", + "element_id": 87 + } + ], + "MAIL_elements": [ + { + "element_name": "mail_protocol_type", + "element_type": "string", + "element_id": 88 + }, + { + "element_name": "mail_account", + "element_type": "string", + "element_id": 89 + }, + { + "element_name": "mail_from_cmd", + "element_type": "string", + "element_id": 90 + }, + { + "element_name": "mail_to_cmd", + "element_type": "string", + "element_id": 91 + }, + { + "element_name": "mail_from", + "element_type": "string", + "element_id": 92 + }, + { + "element_name": "mail_to", + "element_type": "string", + "element_id": 93 + }, + { + "element_name": "mail_cc", + "element_type": "string", + "element_id": 94 + }, + { + "element_name": "mail_bcc", + "element_type": "string", + "element_id": 95 + }, + { + "element_name": "mail_subject", + "element_type": "string", + "element_id": 96 + }, + { + "element_name": "mail_subject_charset", + "element_type": "string", + "element_id": 97 + }, + { + "element_name": "mail_content", + "element_type": "string", + "element_id": 98 + }, + { + "element_name": "mail_content_charset", + "element_type": "string", + "element_id": 99 + }, + { + "element_name": "mail_attachment_name", + "element_type": "string", + "element_id": 100 + }, + { + "element_name": "mail_attachment_name_charset", + "element_type": "string", + "element_id": 101 + }, + { + "element_name": "mail_eml_file", + "element_type": "string", + "element_id": 102 + } + + ], + "DNS_elements": [ + { + "element_name": "dns_message_id", + "element_type": "unsigned32", + "element_id": 103 + }, + { + "element_name": "dns_qr", + "element_type": "unsigned32", + "element_id": 104 + }, + { + "element_name": "dns_opcode", + "element_type": "unsigned32", + "element_id": 105 + }, + { + "element_name": "dns_aa", + "element_type": "unsigned32", + "element_id": 106 + }, + { + "element_name": "dns_tc", + "element_type": "unsigned32", + "element_id": 107 + }, + { + "element_name": "dns_rd", + "element_type": "unsigned32", + "element_id": 108 + }, + { + "element_name": "dns_ra", + "element_type": "unsigned32", + "element_id": 109 + }, + { + "element_name": "dns_rcode", + "element_type": "unsigned32", + "element_id": 110 + }, + { + "element_name": "dns_qdcount", + "element_type": "unsigned32", + "element_id": 111 + }, + { + "element_name": "dns_ancount", + "element_type": "unsigned32", + "element_id": 112 + }, + { + "element_name": "dns_nscount", + "element_type": "unsigned32", + "element_id": 113 + }, + { + "element_name": "dns_arcount", + "element_type": "unsigned32", + "element_id": 114 + }, + { + "element_name": "dns_qname", + "element_type": "string", + "element_id": 115 + }, + { + "element_name": "dns_qtype", + "element_type": "unsigned32", + "element_id": 116 + }, + { + "element_name": "dns_qclass", + "element_type": "unsigned32", + "element_id": 117 + }, + { + "element_name": "dns_cname", + "element_type": "string", + "element_id": 118 + }, + { + "element_name": "dns_sub", + "element_type": "unsigned32", + "element_id": 119 + }, + { + "element_name": "dns_rr", + "element_type": "string", + "element_id": 120 + }, + { + "element_name": "dns_response_latency_ms", + "element_type": "unsigned32", + "element_id": 121 + } + ], + "DTLS_elements": [ + { + "element_name": "dtls_cookie", + "element_type": "string", + "element_id": 133 + }, + { + "element_name": "dtls_version", + "element_type": "string", + "element_id": 134 + }, + { + "element_name": "dtls_sni", + "element_type": "string", + "element_id": 135 + }, + { + "element_name": "dtls_san", + "element_type": "string", + "element_id": 136 + }, + { + "element_name": "dtls_cn", + "element_type": "string", + "element_id": 137 + }, + { + "element_name": "dtls_con_latency_ms", + "element_type": "unsigned32", + "element_id": 138 + }, + { + "element_name": "dtls_ja3_fingerprint", + "element_type": "string", + "element_id": 139 + }, + { + "element_name": "dtls_ja3_hash", + "element_type": "string", + "element_id": 140 + }, + { + "element_name": "dtls_cert_issuer", + "element_type": "string", + "element_id": 141 + }, + { + "element_name": "dtls_cert_subject", + "element_type": "string", + "element_id": 142 + } + ], + "QUIC_elements": [ + { + "element_name": "quic_version", + "element_type": "string", + "element_id": 143 + }, + { + "element_name": "quic_sni", + "element_type": "string", + "element_id": 144 + }, + { + "element_name": "quic_user_agent", + "element_type": "string", + "element_id": 145 + } + ], + "FTP_elements": [ + { + "element_name": "ftp_account", + "element_type": "string", + "element_id": 146 + }, + { + "element_name": "ftp_url", + "element_type": "string", + "element_id": 147 + }, + { + "element_name": "ftp_content", + "element_type": "string", + "element_id": 148 + }, + { + "element_name": "ftp_link_type", + "element_type": "string", + "element_id": 149 + } + ], + "SIP_elements": [ + { + "element_name": "sip_call_id", + "element_type": "string", + "element_id": 150 + }, + { + "element_name": "sip_originator_description", + "element_type": "string", + "element_id": 151 + }, + { + "element_name": "sip_responder_description", + "element_type": "string", + "element_id": 152 + }, + { + "element_name": "sip_user_agent", + "element_type": "string", + "element_id": 153 + }, + { + "element_name": "sip_server", + "element_type": "string", + "element_id": 154 + }, + { + "element_name": "sip_originator_sdp_connect_ip", + "element_type": "string", + "element_id": 155 + }, + { + "element_name": "sip_originator_sdp_media_port", + "element_type": "string", + "element_id": 156 + }, + { + "element_name": "sip_originator_sdp_media_type", + "element_type": "string", + "element_id": 157 + }, + { + "element_name": "sip_originator_sdp_content", + "element_type": "string", + "element_id": 158 + }, + { + "element_name": "sip_responder_sdp_connect_ip", + "element_type": "string", + "element_id": 159 + }, + { + "element_name": "sip_responder_sdp_media_port", + "element_type": "string", + "element_id": 160 + }, + { + "element_name": "sip_responder_sdp_media_type", + "element_type": "string", + "element_id": 161 + }, + { + "element_name": "sip_responder_sdp_content", + "element_type": "string", + "element_id": 162 + }, + { + "element_name": "sip_duration_s", + "element_type": "string", + "element_id": 163 + }, + { + "element_name": "sip_bye", + "element_type": "string", + "element_id": 164 + } + ], + "RTP_elements": [ + { + "element_name": "rtp_payload_type_c2s", + "element_type": "unsigned32", + "element_id": 165 + }, + { + "element_name": "rtp_payload_type_s2c", + "element_type": "unsigned32", + "element_id": 166 + }, + { + "element_name": "rtp_pcap_path", + "element_type": "string", + "element_id": 167 + } + ], + "SSH_elements": [ + { + "element_name": "ssh_version", + "element_type": "string", + "element_id": 168 + }, + { + "element_name": "ssh_auth_success", + "element_type": "string", + "element_id": 169 + }, + { + "element_name": "ssh_client_version", + "element_type": "string", + "element_id": 170 + }, + { + "element_name": "ssh_server_version", + "element_type": "string", + "element_id": 171 + }, + { + "element_name": "ssh_cipher_alg", + "element_type": "string", + "element_id": 172 + }, + { + "element_name": "ssh_mac_alg", + "element_type": "string", + "element_id": 173 + }, + { + "element_name": "ssh_compression_alg", + "element_type": "string", + "element_id": 174 + }, + { + "element_name": "ssh_kex_alg", + "element_type": "string", + "element_id": 175 + }, + { + "element_name": "ssh_host_key_alg", + "element_type": "string", + "element_id": 176 + }, + { + "element_name": "ssh_host_key", + "element_type": "string", + "element_id": 177 + }, + { + "element_name": "ssh_hash", + "element_type": "string", + "element_id": 178 + } + ], + "BGP_elements": [ + { + "element_name": "bgp_message_type", + "element_type": "string", + "element_id": 207 + }, + { + "element_name": "bgp_messages", + "element_type": "string", + "element_id": 208 + } + ], + "RDP_elements": [ + { + "element_name": "rdp_cookie", + "element_type": "string", + "element_id": 213 + }, + { + "element_name": "rdp_security_protocol", + "element_type": "string", + "element_id": 214 + }, + { + "element_name": "rdp_client_channels", + "element_type": "string", + "element_id": 215 + }, + { + "element_name": "rdp_keyboard_layout", + "element_type": "string", + "element_id": 216 + }, + { + "element_name": "rdp_client_version", + "element_type": "string", + "element_id": 217 + }, + { + "element_name": "rdp_client_name", + "element_type": "string", + "element_id": 218 + }, + { + "element_name": "rdp_client_product_id", + "element_type": "string", + "element_id": 219 + }, + { + "element_name": "rdp_desktop_width", + "element_type": "string", + "element_id": 220 + }, + { + "element_name": "rdp_desktop_height", + "element_type": "string", + "element_id": 221 + }, + { + "element_name": "rdp_requested_color_depth", + "element_type": "string", + "element_id": 222 + }, + { + "element_name": "rdp_certificate_type", + "element_type": "unsigned32", + "element_id": 223 + }, + { + "element_name": "rdp_certificate_count", + "element_type": "unsigned32", + "element_id": 224 + }, + { + "element_name": "rdp_certificate_permanent", + "element_type": "string", + "element_id": 225 + }, + { + "element_name": "rdp_encryption_level", + "element_type": "string", + "element_id": 226 + }, + { + "element_name": "rdp_encryption_method", + "element_type": "string", + "element_id": 227 + } + ], + "Tunnels_elements": [ + + ], + "Stratum_elements": [ + { + "element_name": "stratum_cryptocurrency", + "element_type": "unsigned32", + "element_id": 209 + }, + { + "element_name": "stratum_mining_pools", + "element_type": "string", + "element_id": 210 + }, + { + "element_name": "stratum_mining_program", + "element_type": "string", + "element_id": 211 + }, + { + "element_name": "stratum_mining_subscribe", + "element_type": "string", + "element_id": 212 + } + ] +} \ No newline at end of file diff --git a/bin/main.conf b/bin/main.conf index 6f949bb..18d2d27 100644 --- a/bin/main.conf +++ b/bin/main.conf @@ -4,7 +4,7 @@ LOG_PATH="log/master.maat.log" PROFILE="./tsgconf/maat.conf" [TSG_LOG] -MODE=1 +MODE=kafka # kafka or ipfix or (kafka & ipfix) NIC_NAME="lo" LOG_LEVEL=10 LOG_PATH="log/master.kafka" @@ -24,6 +24,8 @@ TCP_MIN_PKTS=3 TCP_MIN_BYTES=5 UDP_MIN_PKTS=3 UDP_MIN_BYTES=5 +IPFIX_EXPORTER_CONF="./tsgconf/ipfix_conf.json" +IPFIX_TEMPLATE_INTERVAL_PKTS=1000 [SECURITY_HITS_METRICS] CYCLE_INTERVAL_MS=1000 diff --git a/ctest/CMakeLists.txt b/ctest/CMakeLists.txt index 5f482e9..ae65e93 100644 --- a/ctest/CMakeLists.txt +++ b/ctest/CMakeLists.txt @@ -5,6 +5,7 @@ project(sapp_ctest) add_test(NAME COPY_CREATE_DIR COMMAND sh -c "mkdir -p ${CMAKE_BINARY_DIR}/testing/tsgconf/ ") add_test(NAME COPY_GTEST_MAIN_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/main.conf ${CMAKE_BINARY_DIR}/testing/tsgconf/") add_test(NAME COPY_GTEST_MAAT_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/maat.conf ${CMAKE_BINARY_DIR}/testing/tsgconf/") +add_test(NAME COPY_GTEST_IPFIX_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/ipfix_conf.json ${CMAKE_BINARY_DIR}/testing/tsgconf/") add_test(NAME COPY_GTEST_LOG_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/tsg_log_field.conf ${CMAKE_BINARY_DIR}/testing/tsgconf/") add_test(NAME COPY_GTEST_PROTO_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/tsg_l7_protocol.conf ${CMAKE_BINARY_DIR}/testing/tsgconf/") add_test(NAME COPY_GTEST_TABLEINFO COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/tsg_static_tableinfo.json ${CMAKE_BINARY_DIR}/testing/tsgconf/") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d48f4a5..df3f4c2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ include_directories(${CMAKE_SOURCE_DIR}/inc) include_directories(/opt/MESA/include/MESA/) include_directories(/usr/include/) -set(TSG_MASTER_DEPEND_DYN_LIB MESA_handle_logger MESA_prof_load maat4 pthread rdkafka cjson MESA_jump_layer fieldstat3 crypto) +set(TSG_MASTER_DEPEND_DYN_LIB MESA_handle_logger MESA_prof_load maat4 pthread rdkafka cjson MESA_jump_layer fieldstat3 crypto ipfix_exporter) set(CMAKE_INSTALL_PREFIX /home/mesasoft/sapp_run) diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 9ea0a41..452fe0a 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -1045,7 +1045,7 @@ int TLD_cancel(struct TLD_handle_t *handle) tsg_stat_log_handle_update(LOG_HANDLE_FREE_CNT, 1); } - + free(handle); handle = NULL; } @@ -1106,7 +1106,6 @@ int TLD_append(struct TLD_handle_t *handle, char *key, void *value, TLD_TYPE typ abort(); default: return -1; - break; } tsg_stat_log_handle_update(LOG_HANDLE_APPEND_CNT, 1); @@ -1168,7 +1167,7 @@ struct TLD_handle_t *TLD_duplicate(struct TLD_handle_t *handle) struct TLD_handle_t *TLD_create(int thread_id) { - if(g_tsg_log_instance->mode==CLOSE) + if(g_tsg_log_instance->mode==CLOSE_SEND_MODE) { return NULL; } @@ -1180,7 +1179,7 @@ struct TLD_handle_t *TLD_create(int thread_id) _handle->document = new Document(_handle->valueAllocator); _handle->document->SetObject(); tsg_stat_log_handle_update(LOG_HANDLE_CREATE_CNT, 1); - + return _handle; } @@ -1987,6 +1986,37 @@ int log_common_fields_new(const char *filename, id2field_t *id2field, struct top return 0; } +static unsigned char tsg_send_mode_get(char *mode_str) +{ + if (mode_str == NULL) + { + return KAFKA_SEND_MODE; // kafka is defualt + } + + unsigned char mode = CLOSE_SEND_MODE; + if (strstr(mode_str, "close") != NULL) + { + return CLOSE_SEND_MODE; + } + + if (strstr(mode_str, "kafka") != NULL) + { + mode |= KAFKA_SEND_MODE; + } + + if (strstr(mode_str, "ipfix") != NULL) + { + mode |= IPFIX_SEND_MODE; + } + + if (mode == CLOSE_SEND_MODE) + { + return KAFKA_SEND_MODE; // kafka is defualt + } + + return mode; +} + struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) { char override_sled_ip[32]={0}; @@ -1997,6 +2027,7 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) struct tsg_log_instance_t *_instance=NULL; char common_field_file[128]={0}; char log_path[128]={0}; + char send_mode[128] = {0}; _instance=(struct tsg_log_instance_t *)calloc(1, sizeof(struct tsg_log_instance_t)); @@ -2028,14 +2059,31 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) return NULL; } - MESA_load_profile_int_def(conffile, "TSG_LOG", "MODE",&(_instance->mode), 0); - if(_instance->mode==CLOSE) + MESA_load_profile_string_def(conffile, "TSG_LOG", "MODE", send_mode, sizeof(send_mode), "kafka"); + _instance->mode = tsg_send_mode_get(send_mode); + + if(_instance->mode==CLOSE_SEND_MODE) { MASTER_LOG(_instance->logger, RLOG_LV_FATAL, LOG_MODULE_SENDLOG, "Disable tsg_send_log"); return _instance; } - MESA_load_profile_int_def(conffile, "TSG_LOG", "RECOVERY_INTERVEL_S", &(_instance->recovery_interval), 30); + if (_instance->mode&IPFIX_SEND_MODE) + { + char ipfix_conf_path[128] = {0}; + MESA_load_profile_string_def(conffile, "TSG_LOG", "IPFIX_EXPORTER_CONF", ipfix_conf_path, sizeof(ipfix_conf_path), "./tsgconf/ipfix_conf.json"); + _instance->ipfix_instance = ipfix_exporter_instance_init(ipfix_conf_path, _instance->logger, get_thread_count()); + if (_instance->ipfix_instance == NULL) + { + free(_instance); + _instance=NULL; + return NULL; + } + + MESA_load_profile_int_def(conffile, "TSG_LOG", "IPFIX_TEMPLATE_INTERVAL_PKTS", &(_instance->ipfix_template_interval_pkts), 1000); + } + + MESA_load_profile_int_def(conffile, "TSG_LOG", "RECOVERY_INTERVEL_S", &(_instance->recovery_interval), 30); MESA_load_profile_string_def(conffile, "TSG_LOG", "COMMON_FIELD_FILE", common_field_file, sizeof(common_field_file), NULL); MESA_load_profile_string_def(conffile, "TSG_LOG", "BROKER_LIST", broker_list, sizeof(broker_list), NULL); @@ -2136,7 +2184,7 @@ void tsg_sendlog_destroy(struct tsg_log_instance_t * instance) return ; } - if(instance->mode!=CLOSE) + if(instance->mode!=CLOSE_SEND_MODE) { for(int i=0; imax_service; i++) { @@ -2170,6 +2218,10 @@ void tsg_sendlog_destroy(struct tsg_log_instance_t * instance) instance->service2topic=NULL; } + if (instance->mode&IPFIX_SEND_MODE) + { + ipfix_exporter_destroy(instance->ipfix_instance); + } MESA_destroy_runtime_log_handle(instance->logger); instance->logger=NULL; @@ -2179,6 +2231,48 @@ void tsg_sendlog_destroy(struct tsg_log_instance_t * instance) return ; } +static int tsg_send_ipfix_message(struct TLD_handle_t *_handle, int thread_id) +{ + if (_handle == NULL) + { + return -1; + } + + if (ipfix_message_get_current_sequence() % g_tsg_log_instance->ipfix_template_interval_pkts == 0) + { + ipfix_message_template_send(g_tsg_log_instance->ipfix_instance, thread_id); + } + + Value::ConstMemberIterator schema_type = _handle->document->FindMember("common_schema_type"); + if (schema_type == _handle->document->MemberEnd()) + { + return -1; + } + + struct ipfix_message* message = ipfix_message_new(g_tsg_log_instance->ipfix_instance, schema_type->value.GetString()); + if (message == NULL) + { + return -1; + } + + for (rapidjson::Value::ConstMemberIterator iter = _handle->document->MemberBegin(); iter != _handle->document->MemberEnd(); ++iter) + { + if (iter->value.GetType() == rapidjson::kStringType) + { + ipfix_message_append(message, iter->name.GetString(), iter->name.GetStringLength(), (char *)iter->value.GetString(), iter->value.GetStringLength()); + } + else if (iter->value.GetType() == rapidjson::kNumberType) + { + int64_t value = iter->value.GetInt64(); + ipfix_message_append(message, iter->name.GetString(), iter->name.GetStringLength(), (char *)&(value), sizeof(int64_t)); + } + } + + ipfix_message_send(g_tsg_log_instance->ipfix_instance, message, (uint16_t)thread_id); + ipfix_message_free(message); + return 0; +} + int send_log_by_type(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, const struct streaminfo *a_stream, LOG_TYPE log_type, int thread_id) { int ret=update_percent(_instance, log_type, LOG_STATUS_DROP, thread_id); @@ -2191,13 +2285,21 @@ int send_log_by_type(struct tsg_log_instance_t *_instance, struct TLD_handle_t * (a_stream==NULL ? "" : printaddr(&(a_stream->addr), thread_id)) ); } - - StringBuffer sb(0, 2048); - Writer writer(sb); - _handle->document->Accept(writer); - - tsg_send_payload(_instance, log_type, (char *)sb.GetString(), sb.GetSize(), thread_id); + if (_instance->mode&KAFKA_SEND_MODE) + { + StringBuffer sb(0, 2048); + Writer writer(sb); + _handle->document->Accept(writer); + + tsg_send_payload(_instance, log_type, (char *)sb.GetString(), sb.GetSize(), thread_id); + } + + if (_instance->mode&IPFIX_SEND_MODE && log_type == LOG_TYPE_SESSION_RECORD) + { + tsg_send_ipfix_message(_handle, thread_id); + } + return 0; } @@ -2320,7 +2422,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl return -1; } - if(_instance->mode==CLOSE) + if(_instance->mode==CLOSE_SEND_MODE) { TLD_cancel(_handle); tsg_stat_sendlog_update(_instance->sum_stat_row_id, LOG_STATUS_DROP, 1); @@ -2396,7 +2498,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl int tsg_register_topic(struct tsg_log_instance_t *instance, char *topic_name) { struct tsg_log_instance_t *_instance=(struct tsg_log_instance_t *)instance; - if(_instance==NULL || _instance->mode==CLOSE || topic_name==NULL || _instance->kafka_handle==NULL) + if(_instance==NULL || _instance->mode==CLOSE_SEND_MODE || topic_name==NULL || _instance->kafka_handle==NULL) { return -1; } @@ -2417,7 +2519,7 @@ int tsg_send_payload(struct tsg_log_instance_t *instance, int topic_id, char *pa int status=0; struct tsg_log_instance_t *_instance=instance; - if(_instance==NULL || _instance->mode==CLOSE) + if(_instance==NULL || _instance->mode==CLOSE_SEND_MODE) { return 0; } diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index 25a9c89..777aaf6 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -1,11 +1,12 @@ #pragma once #include - #include #include #include +#include "ipfix.h" + #define MIN_L7_PROTO_ID 100 #define MAX_L7_PROTO_ID 150 @@ -15,12 +16,9 @@ #define MAX_STRING_LEN32 32 #endif -enum SEND_MODE -{ - CLOSE=0, - KAFKA=1, -}; - +#define CLOSE_SEND_MODE 0x00 +#define KAFKA_SEND_MODE 0x01 +#define IPFIX_SEND_MODE 0x02 enum LOG_FIELD_ID { @@ -177,7 +175,7 @@ struct topic_stat struct tsg_log_instance_t { - int mode; + unsigned char mode; int level; int max_service; int vsystem_id; @@ -198,6 +196,7 @@ struct tsg_log_instance_t int tcp_flow_project_id; int udp_flow_project_id; int sum_stat_row_id; + int ipfix_template_interval_pkts; char tcp_label[MAX_STRING_LEN32]; char udp_label[MAX_STRING_LEN32]; char sasl_username[MAX_STRING_LEN32]; @@ -210,6 +209,7 @@ struct tsg_log_instance_t id2field_t id2field[LOG_COMMON_MAX]; rd_kafka_t *kafka_handle; struct topic_stat *service2topic; + struct ipfix_exporter_instance *ipfix_instance; void *logger; }; diff --git a/test/src/CMakeLists.txt b/test/src/CMakeLists.txt index 2a25309..2a23b58 100644 --- a/test/src/CMakeLists.txt +++ b/test/src/CMakeLists.txt @@ -23,7 +23,7 @@ add_executable(gtest_action ${PROJECT_SOURCE_DIR}/src/tsg_action.cpp ${PROJECT_S target_link_libraries(gtest_action gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger maat4 fieldstat3) add_executable(gtest_sendlog ${PROJECT_SOURCE_DIR}/src/tsg_send_log.cpp ${PROJECT_SOURCE_DIR}/src/tsg_variable.cpp ${PROJECT_SOURCE_DIR}/src/tsg_stat.cpp gtest_common.cpp gtest_kafka.cpp gtest_sendlog.cpp) -target_link_libraries(gtest_sendlog gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger maat4 rdkafka fieldstat3) +target_link_libraries(gtest_sendlog gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger maat4 rdkafka fieldstat3 ipfix_exporter) set(TSG_MASTER_SRC ${PROJECT_SOURCE_DIR}/src/tsg_entry.cpp ${PROJECT_SOURCE_DIR}/src/tsg_rule.cpp @@ -47,15 +47,15 @@ set(TSG_MASTER_SRC ${PROJECT_SOURCE_DIR}/src/tsg_entry.cpp ) add_executable(gtest_master ${TSG_MASTER_SRC} gtest_kafka.cpp gtest_common.cpp gtest_master.cpp) -target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto) +target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter) add_executable(gtest_sync_state ${PROJECT_SOURCE_DIR}/src/tsg_sync_state.cpp ${PROJECT_SOURCE_DIR}/src/mpack.c ${PROJECT_SOURCE_DIR}/src/tsg_stat.cpp gtest_common.cpp gtest_sync_state.cpp) target_link_libraries(gtest_sync_state gtest-static cjson ctemplate-static MESA_prof_load MESA_handle_logger fieldstat3) add_executable(gtest_fieldstat3 ${TSG_MASTER_SRC} gtest_common.cpp gtest_kafka.cpp gtest_fieldstat3.cpp) -target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto) +target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter) add_executable(gtest_proxy ${TSG_MASTER_SRC} gtest_common.cpp gtest_proxy.cpp) -target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap crypto) +target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap crypto ipfix_exporter) file(COPY ../pcap/ DESTINATION ./pcap/)