gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-17290: 支持输出OS fingerprint日志字段(common_client_os_name,common_server_os_name)

Browse Source
This commit is contained in:
刘学利
2023-10-13 07:43:27 +00:00
parent 34834d1a67
commit f7ab3c92ae
12 changed files with 72754 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab-ci.yml
View File

@@ -4,7 +4,7 @@ variables:
BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux"
BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
INSTALL_PREFIX: "/home/mesasoft/sapp_run/"
INSTALL_DEPENDENCY_LIBRARY: systemd-devel libbreakpad_mini numactl-devel zlib-devel vim-common libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaat4 libmaat4-devel quic-devel mesa_sip-devel gtp-devel libMESA_htable-devel libasan mrzcpd rapidjson-devel libMESA_jump_layer-devel stratum-devel rdp-devel dtls-devel-1.0.10.ab7166e libfieldstat3-devel openssl-devel libipfix_exporter
INSTALL_DEPENDENCY_LIBRARY: systemd-devel libbreakpad_mini numactl-devel zlib-devel vim-common libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaat4 libmaat4-devel quic-devel mesa_sip-devel gtp-devel libMESA_htable-devel libasan mrzcpd rapidjson-devel libMESA_jump_layer-devel stratum-devel rdp-devel dtls-devel-1.0.10.ab7166e libfieldstat3-devel openssl-devel libipfix_exporter libosfp-devel libosfp
stages:
- analysis

1
CMakeLists.txt
View File

@@ -84,6 +84,7 @@ install(FILES bin/app_l7_proto_id.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgco
install(FILES bin/tsg_l7_protocol.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/tsg_maat.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/ipfix_conf.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/tsg_osfp_db.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/HTTP200.html DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/HTTP204.html DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)

4
bin/tsg_log_field.conf
View File

@@ -142,4 +142,6 @@ STRING common_in_dest_mac 128
STRING common_out_src_mac 129
STRING common_out_dest_mac 130
LONG common_start_timestamp_ms 131
LONG common_end_timestamp_ms 132
LONG common_end_timestamp_ms 132
STRING common_client_os_name 133
STRING common_server_os_name 134

72594
bin/tsg_osfp_db.json Normal file
View File

File diff suppressed because it is too large Load Diff

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ include_directories(${CMAKE_SOURCE_DIR}/inc)
include_directories(/opt/MESA/include/MESA/)
include_directories(/usr/include/)
set(TSG_MASTER_DEPEND_DYN_LIB MESA_handle_logger MESA_prof_load maat4 pthread rdkafka cjson MESA_jump_layer fieldstat3 crypto ipfix_exporter)
set(TSG_MASTER_DEPEND_DYN_LIB MESA_handle_logger MESA_prof_load maat4 pthread rdkafka cjson MESA_jump_layer fieldstat3 crypto ipfix_exporter osfp)
set(CMAKE_INSTALL_PREFIX /home/mesasoft/sapp_run)

52
src/tsg_bridge.cpp
View File

@@ -236,6 +236,18 @@ void session_runtime_attribute_free(const struct streaminfo *a_stream, int bridg
srt_attribute->proxy_tcp_attr = NULL;
}
if(srt_attribute->client_os!=NULL)
{
dictator_free(a_stream->threadnum, (void *)srt_attribute->client_os);
srt_attribute->client_os=NULL;
}
if(srt_attribute->server_os!=NULL)
{
dictator_free(a_stream->threadnum, (void *)srt_attribute->server_os);
srt_attribute->server_os=NULL;
}
dictator_free(a_stream->threadnum, data);
data=NULL;
}
@@ -370,6 +382,46 @@ int srt_attribute_set_ja3_fingprint(const struct streaminfo *a_stream, const cha
return 0;
}
int srt_attribute_set_client_os(const struct streaminfo *a_stream, const char *os_name)
{
if(os_name==NULL)
{
return 0;
}
struct session_runtime_attribute *srt_attribute=(struct session_runtime_attribute *)session_runtime_attribute_new(a_stream);
if(srt_attribute)
{
size_t os_name_len=strlen(os_name);
srt_attribute->client_os=(char *)dictator_malloc(a_stream->threadnum, os_name_len+1);
memcpy((void *)srt_attribute->client_os, os_name, os_name_len);
srt_attribute->client_os[os_name_len]='\0';
return 1;
}
return 0;
}
int srt_attribute_set_server_os(const struct streaminfo *a_stream, const char *os_name)
{
if(os_name==NULL)
{
return 0;
}
struct session_runtime_attribute *srt_attribute=(struct session_runtime_attribute *)session_runtime_attribute_new(a_stream);
if(srt_attribute)
{
size_t os_name_len=strlen(os_name);
srt_attribute->server_os=(char *)dictator_malloc(a_stream->threadnum, os_name_len+1);
memcpy((void *)srt_attribute->server_os, os_name, os_name_len);
srt_attribute->server_os[os_name_len]='\0';
return 1;
}
return 0;
}
int srt_atttribute_set_umts_user_info(const struct streaminfo *a_stream, struct umts_user_info *user_info)
{
struct session_runtime_attribute *srt_attribute=(struct session_runtime_attribute *)session_runtime_attribute_new(a_stream);

7
src/tsg_bridge.h
View File

@@ -120,7 +120,8 @@ struct session_runtime_action_context
char direction;
char state;
char is_drop_after_n_packets;
char padding[3];
char os_fingerprint_flag;
char padding[2];
enum TSG_METHOD_TYPE method_type;
enum TSG_PROTOCOL protocol;
char *l4_protocol;
@@ -161,6 +162,8 @@ struct session_runtime_attribute
struct tunnel_endpoint *server_endpoint;
unsigned long session_flags;
struct tsg_proxy_tcp_attribute *proxy_tcp_attr;
char *client_os;
char *server_os;
};
@@ -201,6 +204,8 @@ int srt_attribute_set_flags(const struct streaminfo * a_stream, unsigned long s_
int srt_attribute_set_establish_latecy(const struct streaminfo * a_stream);
int srt_attribute_set_ja3_fingprint(const struct streaminfo *a_stream, const char *ja3_fingerprint, int ja3_fingerprint_len);
int srt_attribute_set_reponse_size(const struct streaminfo *a_stream, int http_action_file_size);
int srt_attribute_set_client_os(const struct streaminfo *a_stream, const char *os_name);
int srt_attribute_set_server_os(const struct streaminfo *a_stream, const char *os_name);
void srt_attribute_free_proxy_tcp_option(const struct streaminfo *a_stream);

86
src/tsg_entry.cpp
View File

@@ -1114,6 +1114,65 @@ int session_app_gather_results_update_matched_app_id(const struct streaminfo *a_
return 1;
}
int session_os_fingerprint_generate(const struct streaminfo *a_stream, const void *ip_hdr)
{
struct iphdr *ipv4_hdr=NULL;
struct ip6_hdr *ipv6_hdr=NULL;
struct tcphdr *tcp_hdr=NULL;
struct osfp_result *p_result=NULL;
if(ip_hdr==NULL || a_stream==NULL)
{
return 0;
}
switch(a_stream->addr.addrtype)
{
case ADDR_TYPE_IPV4:
ipv4_hdr=(struct iphdr *)ip_hdr;
tcp_hdr=(struct tcphdr *)MESA_net_jump_to_layer(ipv4_hdr, __ADDR_TYPE_IP_PAIR_V4, ADDR_TYPE_TCP);
if(tcp_hdr->syn)
{
p_result=osfp_ipv4_identify(g_tsg_para.db_osfp, ipv4_hdr, tcp_hdr, tcp_hdr->doff*4);
}
break;
case ADDR_TYPE_IPV6:
ipv6_hdr=(struct ip6_hdr *)ip_hdr;
tcp_hdr=(struct tcphdr *)MESA_net_jump_to_layer(ipv6_hdr, __ADDR_TYPE_IP_PAIR_V6, ADDR_TYPE_TCP);
if(tcp_hdr->syn)
{
p_result=osfp_ipv6_identify(g_tsg_para.db_osfp, ipv6_hdr, tcp_hdr, tcp_hdr->doff*4);
}
break;
default:
return 0;
}
if(p_result!=NULL)
{
const char *os_name=osfp_result_os_name_get(p_result);
if(tcp_hdr->ack)
{
srt_attribute_set_server_os(a_stream, os_name);
}
else
{
srt_attribute_set_client_os(a_stream, os_name);
}
osfp_result_free(p_result);
p_result=NULL;
}
if(!tcp_hdr->syn)
{
return 1;
}
return 0;
}
int session_state_control_packet(const struct streaminfo *a_stream, void *payload, unsigned int payload_len)
{
if(payload==NULL || payload_len==0)
@@ -2555,7 +2614,7 @@ extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, vo
return (state1|state2);
}
extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp, void **pme, int thread_seq, const void *a_packet)
extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp, void **pme, int thread_seq, const void *ip_hdr)
{
struct session_runtime_action_context *srt_action_context=(struct session_runtime_action_context *)(*pme);
@@ -2571,25 +2630,27 @@ extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp,
*pme=(void *)session_runtime_action_context_get(a_tcp);
if(*pme==NULL)
{
srt_action_context=(struct session_runtime_action_context *)session_runtime_action_context_new(a_tcp);
*pme=(void *)srt_action_context;
*pme=(struct session_runtime_action_context *)session_runtime_action_context_new(a_tcp);
}
srt_action_context=(struct session_runtime_action_context *)(*pme);
srt_action_context->direction=get_direction(a_tcp);
srt_action_context->last_update_metric_time = tsg_get_current_time_ms();
}
if(srt_action_context->set_latency_flag==0)
{
srt_action_context->set_latency_flag=session_tcp_establish_latency_ms_set(a_tcp, thread_seq, a_packet);
srt_action_context->set_latency_flag=session_tcp_establish_latency_ms_set(a_tcp, thread_seq, ip_hdr);
}
if (a_packet!=NULL)
if(srt_action_context->os_fingerprint_flag==0)
{
tsg_proxy_tcp_options_parse(a_tcp, a_packet);
srt_action_context->os_fingerprint_flag=session_os_fingerprint_generate(a_tcp, ip_hdr);
}
unsigned char state=tsg_master_all_entry(a_tcp, a_tcp->pktstate, pme, thread_seq, a_packet);
tsg_proxy_tcp_options_parse(a_tcp, ip_hdr);
unsigned char state=tsg_master_all_entry(a_tcp, a_tcp->pktstate, pme, thread_seq, ip_hdr);
if(state&APP_STATE_DROPME || a_tcp->pktstate==OP_STATE_CLOSE)
{
@@ -2721,6 +2782,15 @@ extern "C" int TSG_MASTER_INIT()
g_tsg_para.send_resetall=0;
char osfp_db_json_path[256]={0};
MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "OSFP_DB_JSON_PATH", osfp_db_json_path, sizeof(osfp_db_json_path), "tsgconf/tsg_osfp_db.json");
g_tsg_para.db_osfp=osfp_db_new((const char *)osfp_db_json_path);
if(g_tsg_para.db_osfp==NULL)
{
MASTER_LOG(g_tsg_para.logger, RLOG_LV_FATAL, LOG_MODULE_INIT, "osfp_db_new failed, please check %s", osfp_db_json_path);
return -1;
}
return 0;
}

10
src/tsg_send_log.cpp
View File

@@ -1727,6 +1727,16 @@ int set_session_attributes(struct tsg_log_instance_t *_instance, struct TLD_hand
TLD_append(_handle, _instance->id2field[LOG_COMMON_TUNNELS_ENDPOINT_B_DESC].name, (void *)srt_attribute->server_endpoint->description, TLD_TYPE_STRING);
}
if(srt_attribute->client_os!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLIENT_OS_NAME].name, (void *)srt_attribute->client_os, TLD_TYPE_STRING);
}
if(srt_attribute->server_os!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_OS_NAME].name, (void *)srt_attribute->server_os, TLD_TYPE_STRING);
}
return 1;
}

2
src/tsg_send_log_internal.h
View File

@@ -154,6 +154,8 @@ enum LOG_FIELD_ID
LOG_COMMON_OUTCOMING_DST_MAC,
LOG_COMMON_START_TIME_MS,
LOG_COMMON_END_TIME_MS,
LOG_COMMON_CLIENT_OS_NAME,
LOG_COMMON_SERVER_OS_NAME,
LOG_COMMON_MAX
};

4
src/tsg_variable.h
View File

@@ -3,7 +3,8 @@
#include "uthash.h"
#include "tsg_bridge.h"
#include <MESA/stream.h>
#include <osfp.h>
#include <stream.h>
#include <ctemplate/template.h>
enum DEPLOY_MODE
@@ -91,6 +92,7 @@ struct tsg_rt_para
ctemplate::Template *tpl_403,*tpl_404;
ctemplate::Template *tpl_200,*tpl_204;
ctemplate::Template *tpl_303;
struct osfp_db *db_osfp;
};
extern struct tsg_rt_para g_tsg_para;;

6
test/src/CMakeLists.txt
View File

@@ -47,15 +47,15 @@ set(TSG_MASTER_SRC ${PROJECT_SOURCE_DIR}/src/tsg_entry.cpp
)
add_executable(gtest_master ${TSG_MASTER_SRC} gtest_kafka.cpp gtest_common.cpp gtest_master.cpp)
target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter)
target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter osfp)
add_executable(gtest_sync_state ${PROJECT_SOURCE_DIR}/src/tsg_sync_state.cpp ${PROJECT_SOURCE_DIR}/src/mpack.c ${PROJECT_SOURCE_DIR}/src/tsg_stat.cpp gtest_common.cpp gtest_sync_state.cpp)
target_link_libraries(gtest_sync_state gtest-static cjson ctemplate-static MESA_prof_load MESA_handle_logger fieldstat3)
add_executable(gtest_fieldstat3 ${TSG_MASTER_SRC} gtest_common.cpp gtest_kafka.cpp gtest_fieldstat3.cpp)
target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter)
target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter osfp)
add_executable(gtest_proxy ${TSG_MASTER_SRC} gtest_common.cpp gtest_proxy.cpp)
target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap crypto ipfix_exporter)
target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap crypto ipfix_exporter osfp)
file(COPY ../pcap/ DESTINATION ./pcap/)