TSG-17290: 支持输出OS fingerprint日志字段(common_client_os_name,common_server_os_name)

.gitlab-ci.yml

@@ -4,7 +4,7 @@ variables:
   BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux"
   BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
   INSTALL_PREFIX: "/home/mesasoft/sapp_run/"
-  INSTALL_DEPENDENCY_LIBRARY: systemd-devel libbreakpad_mini numactl-devel zlib-devel vim-common libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaat4 libmaat4-devel quic-devel mesa_sip-devel gtp-devel libMESA_htable-devel libasan mrzcpd rapidjson-devel libMESA_jump_layer-devel stratum-devel rdp-devel dtls-devel-1.0.10.ab7166e libfieldstat3-devel openssl-devel libipfix_exporter
+  INSTALL_DEPENDENCY_LIBRARY: systemd-devel libbreakpad_mini numactl-devel zlib-devel vim-common libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaat4 libmaat4-devel quic-devel mesa_sip-devel gtp-devel libMESA_htable-devel libasan mrzcpd rapidjson-devel libMESA_jump_layer-devel stratum-devel rdp-devel dtls-devel-1.0.10.ab7166e libfieldstat3-devel openssl-devel libipfix_exporter libosfp-devel libosfp
 
 stages:
 - analysis

CMakeLists.txt

@@ -84,6 +84,7 @@ install(FILES bin/app_l7_proto_id.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgco
 install(FILES bin/tsg_l7_protocol.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
 install(FILES bin/tsg_maat.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
 install(FILES bin/ipfix_conf.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
+install(FILES bin/tsg_osfp_db.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
 
 install(FILES bin/HTTP200.html DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
 install(FILES bin/HTTP204.html DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)

bin/tsg_log_field.conf

@@ -143,3 +143,5 @@ STRING		common_out_src_mac		129
 STRING		common_out_dest_mac		130
 LONG		common_start_timestamp_ms   131
 LONG		common_end_timestamp_ms		132
+STRING		common_client_os_name	133
+STRING		common_server_os_name 	134

src/CMakeLists.txt

@@ -8,7 +8,7 @@ include_directories(${CMAKE_SOURCE_DIR}/inc)
 include_directories(/opt/MESA/include/MESA/)
 include_directories(/usr/include/)
 
-set(TSG_MASTER_DEPEND_DYN_LIB  MESA_handle_logger MESA_prof_load maat4 pthread rdkafka cjson MESA_jump_layer fieldstat3 crypto ipfix_exporter)
+set(TSG_MASTER_DEPEND_DYN_LIB  MESA_handle_logger MESA_prof_load maat4 pthread rdkafka cjson MESA_jump_layer fieldstat3 crypto ipfix_exporter osfp)
 
 set(CMAKE_INSTALL_PREFIX /home/mesasoft/sapp_run)
 

src/tsg_bridge.cpp

@@ -236,6 +236,18 @@ void session_runtime_attribute_free(const struct streaminfo *a_stream, int bridg
 			srt_attribute->proxy_tcp_attr = NULL;
 		}
 
+		if(srt_attribute->client_os!=NULL)
+		{
+			dictator_free(a_stream->threadnum, (void *)srt_attribute->client_os);
+			srt_attribute->client_os=NULL;
+		}
+
+		if(srt_attribute->server_os!=NULL)
+		{
+			dictator_free(a_stream->threadnum, (void *)srt_attribute->server_os);
+			srt_attribute->server_os=NULL;
+		}
+
 		dictator_free(a_stream->threadnum, data);
 		data=NULL;
 	}
@@ -370,6 +382,46 @@ int srt_attribute_set_ja3_fingprint(const struct streaminfo *a_stream, const cha
 	return 0;
 }
 
+int srt_attribute_set_client_os(const struct streaminfo *a_stream, const char *os_name)
+{
+	if(os_name==NULL)
+	{
+		return 0;
+	}
+
+	struct session_runtime_attribute *srt_attribute=(struct session_runtime_attribute *)session_runtime_attribute_new(a_stream);
+	if(srt_attribute)
+	{
+		size_t os_name_len=strlen(os_name);
+		srt_attribute->client_os=(char *)dictator_malloc(a_stream->threadnum, os_name_len+1);
+		memcpy((void *)srt_attribute->client_os, os_name, os_name_len);
+		srt_attribute->client_os[os_name_len]='\0';
+		return 1;
+	}
+
+	return 0;
+}
+
+int srt_attribute_set_server_os(const struct streaminfo *a_stream, const char *os_name)
+{
+	if(os_name==NULL)
+	{
+		return 0;
+	}
+
+	struct session_runtime_attribute *srt_attribute=(struct session_runtime_attribute *)session_runtime_attribute_new(a_stream);
+	if(srt_attribute)
+	{
+		size_t os_name_len=strlen(os_name);
+		srt_attribute->server_os=(char *)dictator_malloc(a_stream->threadnum, os_name_len+1);
+		memcpy((void *)srt_attribute->server_os, os_name, os_name_len);
+		srt_attribute->server_os[os_name_len]='\0';
+		return 1;
+	}
+
+	return 0;
+}
+
 int srt_atttribute_set_umts_user_info(const struct streaminfo *a_stream, struct umts_user_info *user_info)
 {
 	struct session_runtime_attribute *srt_attribute=(struct session_runtime_attribute *)session_runtime_attribute_new(a_stream);

src/tsg_bridge.h

@@ -120,7 +120,8 @@ struct session_runtime_action_context
 	char direction;
 	char state;
 	char is_drop_after_n_packets;
-	char padding[3];
+	char os_fingerprint_flag;
+	char padding[2];
 	enum TSG_METHOD_TYPE method_type; 
 	enum TSG_PROTOCOL protocol;
 	char *l4_protocol;
@@ -161,6 +162,8 @@ struct session_runtime_attribute
 	struct tunnel_endpoint *server_endpoint;
 	unsigned long session_flags;
 	struct tsg_proxy_tcp_attribute *proxy_tcp_attr;
+	char *client_os;
+	char *server_os;
 };
 
 
@@ -201,6 +204,8 @@ int srt_attribute_set_flags(const struct streaminfo * a_stream, unsigned long s_
 int srt_attribute_set_establish_latecy(const struct streaminfo * a_stream);
 int srt_attribute_set_ja3_fingprint(const struct streaminfo *a_stream, const char *ja3_fingerprint, int ja3_fingerprint_len);
 int srt_attribute_set_reponse_size(const struct streaminfo *a_stream, int http_action_file_size);
+int srt_attribute_set_client_os(const struct streaminfo *a_stream, const char *os_name);
+int srt_attribute_set_server_os(const struct streaminfo *a_stream, const char *os_name);
 
 void srt_attribute_free_proxy_tcp_option(const struct streaminfo *a_stream);
 

src/tsg_entry.cpp

@@ -1114,6 +1114,65 @@ int session_app_gather_results_update_matched_app_id(const struct streaminfo *a_
 	return 1;
 }
 
+int session_os_fingerprint_generate(const struct streaminfo *a_stream, const void *ip_hdr)
+{
+	struct iphdr *ipv4_hdr=NULL;
+	struct ip6_hdr *ipv6_hdr=NULL;
+	struct tcphdr  *tcp_hdr=NULL;
+    struct osfp_result *p_result=NULL;
+
+	if(ip_hdr==NULL || a_stream==NULL)
+	{
+		return 0;
+	}
+	
+	switch(a_stream->addr.addrtype)
+	{
+		case	ADDR_TYPE_IPV4:
+			ipv4_hdr=(struct iphdr *)ip_hdr;
+			tcp_hdr=(struct tcphdr *)MESA_net_jump_to_layer(ipv4_hdr, __ADDR_TYPE_IP_PAIR_V4, ADDR_TYPE_TCP);
+			if(tcp_hdr->syn)
+			{
+				p_result=osfp_ipv4_identify(g_tsg_para.db_osfp, ipv4_hdr, tcp_hdr, tcp_hdr->doff*4);
+			}
+            		
+			break;
+		case	ADDR_TYPE_IPV6:	
+			ipv6_hdr=(struct ip6_hdr *)ip_hdr;
+			tcp_hdr=(struct tcphdr *)MESA_net_jump_to_layer(ipv6_hdr, __ADDR_TYPE_IP_PAIR_V6, ADDR_TYPE_TCP);
+			if(tcp_hdr->syn)
+			{
+            	p_result=osfp_ipv6_identify(g_tsg_para.db_osfp, ipv6_hdr, tcp_hdr, tcp_hdr->doff*4);
+			}
+			break;
+		default:
+			return 0;
+	}
+	
+	if(p_result!=NULL)
+	{
+		const char *os_name=osfp_result_os_name_get(p_result);
+		if(tcp_hdr->ack)
+		{
+        	srt_attribute_set_server_os(a_stream, os_name);
+		}
+		else
+		{
+			srt_attribute_set_client_os(a_stream, os_name);
+		}
+
+		osfp_result_free(p_result);
+		p_result=NULL;
+	}
+
+	if(!tcp_hdr->syn)
+	{
+		return 1;
+	}
+
+	return 0;
+}
+
 int session_state_control_packet(const struct streaminfo *a_stream, void *payload, unsigned int payload_len)
 {
 	if(payload==NULL || payload_len==0)
@@ -2555,7 +2614,7 @@ extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, vo
 	return (state1|state2);
 }
 
-extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp, void **pme, int thread_seq, const void *a_packet)
+extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp, void **pme, int thread_seq, const void *ip_hdr)
 {
 	struct session_runtime_action_context *srt_action_context=(struct session_runtime_action_context *)(*pme);
 
@@ -2571,25 +2630,27 @@ extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp,
 		*pme=(void *)session_runtime_action_context_get(a_tcp);
 		if(*pme==NULL)
 		{
-			srt_action_context=(struct session_runtime_action_context *)session_runtime_action_context_new(a_tcp);
+			*pme=(struct session_runtime_action_context *)session_runtime_action_context_new(a_tcp);
-			*pme=(void *)srt_action_context;
 		}
 		
+		srt_action_context=(struct session_runtime_action_context *)(*pme);
 		srt_action_context->direction=get_direction(a_tcp);
 		srt_action_context->last_update_metric_time = tsg_get_current_time_ms();
 	}
 
 	if(srt_action_context->set_latency_flag==0)
 	{
-		srt_action_context->set_latency_flag=session_tcp_establish_latency_ms_set(a_tcp, thread_seq, a_packet);
+		srt_action_context->set_latency_flag=session_tcp_establish_latency_ms_set(a_tcp, thread_seq, ip_hdr);
 	}
 
-	if (a_packet!=NULL)
+	if(srt_action_context->os_fingerprint_flag==0)
 	{
-		tsg_proxy_tcp_options_parse(a_tcp, a_packet);
+		srt_action_context->os_fingerprint_flag=session_os_fingerprint_generate(a_tcp, ip_hdr);
 	}
 
-	unsigned char state=tsg_master_all_entry(a_tcp, a_tcp->pktstate, pme, thread_seq, a_packet);
+	tsg_proxy_tcp_options_parse(a_tcp, ip_hdr);
+
+	unsigned char state=tsg_master_all_entry(a_tcp, a_tcp->pktstate, pme, thread_seq, ip_hdr);
 
 	if(state&APP_STATE_DROPME || a_tcp->pktstate==OP_STATE_CLOSE)
 	{
@@ -2721,6 +2782,15 @@ extern "C" int TSG_MASTER_INIT()
 
 	g_tsg_para.send_resetall=0;
 
+	char osfp_db_json_path[256]={0};
+	MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "OSFP_DB_JSON_PATH", osfp_db_json_path, sizeof(osfp_db_json_path), "tsgconf/tsg_osfp_db.json");
+	g_tsg_para.db_osfp=osfp_db_new((const char *)osfp_db_json_path);
+	if(g_tsg_para.db_osfp==NULL)
+	{
+		MASTER_LOG(g_tsg_para.logger, RLOG_LV_FATAL, LOG_MODULE_INIT, "osfp_db_new failed, please check %s", osfp_db_json_path);
+		return -1;
+	}
+
 	return 0;
 }
 

src/tsg_send_log.cpp

@@ -1727,6 +1727,16 @@ int set_session_attributes(struct tsg_log_instance_t *_instance, struct TLD_hand
 		TLD_append(_handle, _instance->id2field[LOG_COMMON_TUNNELS_ENDPOINT_B_DESC].name, (void *)srt_attribute->server_endpoint->description, TLD_TYPE_STRING);
 	}
 
+	if(srt_attribute->client_os!=NULL)
+	{
+		TLD_append(_handle, _instance->id2field[LOG_COMMON_CLIENT_OS_NAME].name, (void *)srt_attribute->client_os, TLD_TYPE_STRING);
+	}
+
+	if(srt_attribute->server_os!=NULL)
+	{
+		TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_OS_NAME].name, (void *)srt_attribute->server_os, TLD_TYPE_STRING);
+	}
+
 	return 1;
 }
 

src/tsg_send_log_internal.h

@@ -154,6 +154,8 @@ enum LOG_FIELD_ID
 	LOG_COMMON_OUTCOMING_DST_MAC,
 	LOG_COMMON_START_TIME_MS,
 	LOG_COMMON_END_TIME_MS,
+	LOG_COMMON_CLIENT_OS_NAME,
+	LOG_COMMON_SERVER_OS_NAME,
 	LOG_COMMON_MAX
 };
 

src/tsg_variable.h

@@ -3,7 +3,8 @@
 #include "uthash.h"
 #include "tsg_bridge.h"
 
-#include <MESA/stream.h>
+#include <osfp.h>
+#include <stream.h>
 #include <ctemplate/template.h>
 
 enum DEPLOY_MODE
@@ -91,6 +92,7 @@ struct tsg_rt_para
 	ctemplate::Template *tpl_403,*tpl_404;
 	ctemplate::Template *tpl_200,*tpl_204;
 	ctemplate::Template *tpl_303;
+	struct osfp_db *db_osfp;
 };
 
 extern struct tsg_rt_para g_tsg_para;;

test/src/CMakeLists.txt

@@ -47,15 +47,15 @@ set(TSG_MASTER_SRC ${PROJECT_SOURCE_DIR}/src/tsg_entry.cpp
 )
 
 add_executable(gtest_master ${TSG_MASTER_SRC} gtest_kafka.cpp gtest_common.cpp gtest_master.cpp)
-target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter)
+target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter osfp)
 
 add_executable(gtest_sync_state ${PROJECT_SOURCE_DIR}/src/tsg_sync_state.cpp ${PROJECT_SOURCE_DIR}/src/mpack.c ${PROJECT_SOURCE_DIR}/src/tsg_stat.cpp gtest_common.cpp gtest_sync_state.cpp)
 target_link_libraries(gtest_sync_state gtest-static cjson ctemplate-static MESA_prof_load MESA_handle_logger fieldstat3)
 
 add_executable(gtest_fieldstat3 ${TSG_MASTER_SRC} gtest_common.cpp gtest_kafka.cpp gtest_fieldstat3.cpp)
-target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter)
+target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter osfp)
 
 add_executable(gtest_proxy ${TSG_MASTER_SRC} gtest_common.cpp gtest_proxy.cpp)
-target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap crypto ipfix_exporter)
+target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap crypto ipfix_exporter osfp)
 
 file(COPY ../pcap/ DESTINATION ./pcap/)