gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-15739:功能端支持输出IPFIX封装的UDP报文

Browse Source
This commit is contained in:
杨玉波
2023-09-01 08:50:24 +00:00
committed by 刘学利
parent fc4c49379f
commit 5c1e250c7a
9 changed files with 1281 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab-ci.yml
View File

@@ -4,7 +4,7 @@ variables:
BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux"
BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
INSTALL_PREFIX: "/home/mesasoft/sapp_run/"
INSTALL_DEPENDENCY_LIBRARY: systemd-devel libbreakpad_mini numactl-devel zlib-devel vim-common libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaat4-4.0.31.73887d5 libmaat4-devel-4.0.31.73887d5 quic-devel mesa_sip-devel gtp-devel libMESA_htable-devel libasan mrzcpd rapidjson-devel libMESA_jump_layer-devel stratum-devel rdp-devel dtls-devel-1.0.10.ab7166e libfieldstat3-devel openssl-devel
INSTALL_DEPENDENCY_LIBRARY: systemd-devel libbreakpad_mini numactl-devel zlib-devel vim-common libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaat4-4.0.31.73887d5 libmaat4-devel-4.0.31.73887d5 quic-devel mesa_sip-devel gtp-devel libMESA_htable-devel libasan mrzcpd rapidjson-devel libMESA_jump_layer-devel stratum-devel rdp-devel dtls-devel-1.0.10.ab7166e libfieldstat3-devel openssl-devel libipfix_exporter
stages:
- analysis

1
CMakeLists.txt
View File

@@ -83,6 +83,7 @@ install(FILES bin/tsg_log_field.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf
install(FILES bin/app_l7_proto_id.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/tsg_l7_protocol.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/tsg_maat.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/ipfix_conf.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/HTTP200.html DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
install(FILES bin/HTTP204.html DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)

1143
bin/ipfix_conf.json Normal file
View File

File diff suppressed because it is too large Load Diff

4
bin/main.conf
View File

@@ -4,7 +4,7 @@ LOG_PATH="log/master.maat.log"
PROFILE="./tsgconf/maat.conf"
[TSG_LOG]
MODE=1
MODE=kafka # kafka or ipfix or (kafka & ipfix)
NIC_NAME="lo"
LOG_LEVEL=10
LOG_PATH="log/master.kafka"
@@ -24,6 +24,8 @@ TCP_MIN_PKTS=3
TCP_MIN_BYTES=5
UDP_MIN_PKTS=3
UDP_MIN_BYTES=5
IPFIX_EXPORTER_CONF="./tsgconf/ipfix_conf.json"
IPFIX_TEMPLATE_INTERVAL_PKTS=1000
[SECURITY_HITS_METRICS]
CYCLE_INTERVAL_MS=1000

1
ctest/CMakeLists.txt
View File

@@ -5,6 +5,7 @@ project(sapp_ctest)
add_test(NAME COPY_CREATE_DIR COMMAND sh -c "mkdir -p ${CMAKE_BINARY_DIR}/testing/tsgconf/ ")
add_test(NAME COPY_GTEST_MAIN_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/main.conf ${CMAKE_BINARY_DIR}/testing/tsgconf/")
add_test(NAME COPY_GTEST_MAAT_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/maat.conf ${CMAKE_BINARY_DIR}/testing/tsgconf/")
add_test(NAME COPY_GTEST_IPFIX_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/ipfix_conf.json ${CMAKE_BINARY_DIR}/testing/tsgconf/")
add_test(NAME COPY_GTEST_LOG_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/tsg_log_field.conf ${CMAKE_BINARY_DIR}/testing/tsgconf/")
add_test(NAME COPY_GTEST_PROTO_CONF COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/tsg_l7_protocol.conf ${CMAKE_BINARY_DIR}/testing/tsgconf/")
add_test(NAME COPY_GTEST_TABLEINFO COMMAND sh -c "cp -r ${CMAKE_SOURCE_DIR}/bin/tsg_static_tableinfo.json ${CMAKE_BINARY_DIR}/testing/tsgconf/")

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ include_directories(${CMAKE_SOURCE_DIR}/inc)
include_directories(/opt/MESA/include/MESA/)
include_directories(/usr/include/)
set(TSG_MASTER_DEPEND_DYN_LIB MESA_handle_logger MESA_prof_load maat4 pthread rdkafka cjson MESA_jump_layer fieldstat3 crypto)
set(TSG_MASTER_DEPEND_DYN_LIB MESA_handle_logger MESA_prof_load maat4 pthread rdkafka cjson MESA_jump_layer fieldstat3 crypto ipfix_exporter)
set(CMAKE_INSTALL_PREFIX /home/mesasoft/sapp_run)

136
src/tsg_send_log.cpp
View File

@@ -1045,7 +1045,7 @@ int TLD_cancel(struct TLD_handle_t *handle)
tsg_stat_log_handle_update(LOG_HANDLE_FREE_CNT, 1);
}
free(handle);
handle = NULL;
}
@@ -1106,7 +1106,6 @@ int TLD_append(struct TLD_handle_t *handle, char *key, void *value, TLD_TYPE typ
abort();
default:
return -1;
break;
}
tsg_stat_log_handle_update(LOG_HANDLE_APPEND_CNT, 1);
@@ -1168,7 +1167,7 @@ struct TLD_handle_t *TLD_duplicate(struct TLD_handle_t *handle)
struct TLD_handle_t *TLD_create(int thread_id)
{
if(g_tsg_log_instance->mode==CLOSE)
if(g_tsg_log_instance->mode==CLOSE_SEND_MODE)
{
return NULL;
}
@@ -1180,7 +1179,7 @@ struct TLD_handle_t *TLD_create(int thread_id)
_handle->document = new Document(_handle->valueAllocator);
_handle->document->SetObject();
tsg_stat_log_handle_update(LOG_HANDLE_CREATE_CNT, 1);
return _handle;
}
@@ -1987,6 +1986,37 @@ int log_common_fields_new(const char *filename, id2field_t *id2field, struct top
return 0;
}
static unsigned char tsg_send_mode_get(char *mode_str)
{
if (mode_str == NULL)
{
return KAFKA_SEND_MODE; // kafka is defualt
}
unsigned char mode = CLOSE_SEND_MODE;
if (strstr(mode_str, "close") != NULL)
{
return CLOSE_SEND_MODE;
}
if (strstr(mode_str, "kafka") != NULL)
{
mode |= KAFKA_SEND_MODE;
}
if (strstr(mode_str, "ipfix") != NULL)
{
mode |= IPFIX_SEND_MODE;
}
if (mode == CLOSE_SEND_MODE)
{
return KAFKA_SEND_MODE; // kafka is defualt
}
return mode;
}
struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
{
char override_sled_ip[32]={0};
@@ -1997,6 +2027,7 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
struct tsg_log_instance_t *_instance=NULL;
char common_field_file[128]={0};
char log_path[128]={0};
char send_mode[128] = {0};
_instance=(struct tsg_log_instance_t *)calloc(1, sizeof(struct tsg_log_instance_t));
@@ -2028,14 +2059,31 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
return NULL;
}
MESA_load_profile_int_def(conffile, "TSG_LOG", "MODE",&(_instance->mode), 0);
if(_instance->mode==CLOSE)
MESA_load_profile_string_def(conffile, "TSG_LOG", "MODE", send_mode, sizeof(send_mode), "kafka");
_instance->mode = tsg_send_mode_get(send_mode);
if(_instance->mode==CLOSE_SEND_MODE)
{
MASTER_LOG(_instance->logger, RLOG_LV_FATAL, LOG_MODULE_SENDLOG, "Disable tsg_send_log");
return _instance;
}
MESA_load_profile_int_def(conffile, "TSG_LOG", "RECOVERY_INTERVEL_S", &(_instance->recovery_interval), 30);
if (_instance->mode&IPFIX_SEND_MODE)
{
char ipfix_conf_path[128] = {0};
MESA_load_profile_string_def(conffile, "TSG_LOG", "IPFIX_EXPORTER_CONF", ipfix_conf_path, sizeof(ipfix_conf_path), "./tsgconf/ipfix_conf.json");
_instance->ipfix_instance = ipfix_exporter_instance_init(ipfix_conf_path, _instance->logger, get_thread_count());
if (_instance->ipfix_instance == NULL)
{
free(_instance);
_instance=NULL;
return NULL;
}
MESA_load_profile_int_def(conffile, "TSG_LOG", "IPFIX_TEMPLATE_INTERVAL_PKTS", &(_instance->ipfix_template_interval_pkts), 1000);
}
MESA_load_profile_int_def(conffile, "TSG_LOG", "RECOVERY_INTERVEL_S", &(_instance->recovery_interval), 30);
MESA_load_profile_string_def(conffile, "TSG_LOG", "COMMON_FIELD_FILE", common_field_file, sizeof(common_field_file), NULL);
MESA_load_profile_string_def(conffile, "TSG_LOG", "BROKER_LIST", broker_list, sizeof(broker_list), NULL);
@@ -2136,7 +2184,7 @@ void tsg_sendlog_destroy(struct tsg_log_instance_t * instance)
return ;
}
if(instance->mode!=CLOSE)
if(instance->mode!=CLOSE_SEND_MODE)
{
for(int i=0; i<instance->max_service; i++)
{
@@ -2170,6 +2218,10 @@ void tsg_sendlog_destroy(struct tsg_log_instance_t * instance)
instance->service2topic=NULL;
}
if (instance->mode&IPFIX_SEND_MODE)
{
ipfix_exporter_destroy(instance->ipfix_instance);
}
MESA_destroy_runtime_log_handle(instance->logger);
instance->logger=NULL;
@@ -2179,6 +2231,48 @@ void tsg_sendlog_destroy(struct tsg_log_instance_t * instance)
return ;
}
static int tsg_send_ipfix_message(struct TLD_handle_t *_handle, int thread_id)
{
if (_handle == NULL)
{
return -1;
}
if (ipfix_message_get_current_sequence() % g_tsg_log_instance->ipfix_template_interval_pkts == 0)
{
ipfix_message_template_send(g_tsg_log_instance->ipfix_instance, thread_id);
}
Value::ConstMemberIterator schema_type = _handle->document->FindMember("common_schema_type");
if (schema_type == _handle->document->MemberEnd())
{
return -1;
}
struct ipfix_message* message = ipfix_message_new(g_tsg_log_instance->ipfix_instance, schema_type->value.GetString());
if (message == NULL)
{
return -1;
}
for (rapidjson::Value::ConstMemberIterator iter = _handle->document->MemberBegin(); iter != _handle->document->MemberEnd(); ++iter)
{
if (iter->value.GetType() == rapidjson::kStringType)
{
ipfix_message_append(message, iter->name.GetString(), iter->name.GetStringLength(), (char *)iter->value.GetString(), iter->value.GetStringLength());
}
else if (iter->value.GetType() == rapidjson::kNumberType)
{
int64_t value = iter->value.GetInt64();
ipfix_message_append(message, iter->name.GetString(), iter->name.GetStringLength(), (char *)&(value), sizeof(int64_t));
}
}
ipfix_message_send(g_tsg_log_instance->ipfix_instance, message, (uint16_t)thread_id);
ipfix_message_free(message);
return 0;
}
int send_log_by_type(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, const struct streaminfo *a_stream, LOG_TYPE log_type, int thread_id)
{
int ret=update_percent(_instance, log_type, LOG_STATUS_DROP, thread_id);
@@ -2191,13 +2285,21 @@ int send_log_by_type(struct tsg_log_instance_t *_instance, struct TLD_handle_t *
(a_stream==NULL ? "" : printaddr(&(a_stream->addr), thread_id))
);
}
StringBuffer sb(0, 2048);
Writer<StringBuffer> writer(sb);
_handle->document->Accept(writer);
tsg_send_payload(_instance, log_type, (char *)sb.GetString(), sb.GetSize(), thread_id);
if (_instance->mode&KAFKA_SEND_MODE)
{
StringBuffer sb(0, 2048);
Writer<StringBuffer> writer(sb);
_handle->document->Accept(writer);
tsg_send_payload(_instance, log_type, (char *)sb.GetString(), sb.GetSize(), thread_id);
}
if (_instance->mode&IPFIX_SEND_MODE && log_type == LOG_TYPE_SESSION_RECORD)
{
tsg_send_ipfix_message(_handle, thread_id);
}
return 0;
}
@@ -2320,7 +2422,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
return -1;
}
if(_instance->mode==CLOSE)
if(_instance->mode==CLOSE_SEND_MODE)
{
TLD_cancel(_handle);
tsg_stat_sendlog_update(_instance->sum_stat_row_id, LOG_STATUS_DROP, 1);
@@ -2396,7 +2498,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
int tsg_register_topic(struct tsg_log_instance_t *instance, char *topic_name)
{
struct tsg_log_instance_t *_instance=(struct tsg_log_instance_t *)instance;
if(_instance==NULL || _instance->mode==CLOSE || topic_name==NULL || _instance->kafka_handle==NULL)
if(_instance==NULL || _instance->mode==CLOSE_SEND_MODE || topic_name==NULL || _instance->kafka_handle==NULL)
{
return -1;
}
@@ -2417,7 +2519,7 @@ int tsg_send_payload(struct tsg_log_instance_t *instance, int topic_id, char *pa
int status=0;
struct tsg_log_instance_t *_instance=instance;
if(_instance==NULL || _instance->mode==CLOSE)
if(_instance==NULL || _instance->mode==CLOSE_SEND_MODE)
{
return 0;
}

16
src/tsg_send_log_internal.h
View File

@@ -1,11 +1,12 @@
#pragma once
#include <MESA/field_stat2.h>
#include <librdkafka/rdkafka.h>
#include <MESA/cJSON.h>
#include <time.h>
#include "ipfix.h"
#define MIN_L7_PROTO_ID 100
#define MAX_L7_PROTO_ID 150
@@ -15,12 +16,9 @@
#define MAX_STRING_LEN32 32
#endif
enum SEND_MODE
{
CLOSE=0,
KAFKA=1,
};
#define CLOSE_SEND_MODE 0x00
#define KAFKA_SEND_MODE 0x01
#define IPFIX_SEND_MODE 0x02
enum LOG_FIELD_ID
{
@@ -177,7 +175,7 @@ struct topic_stat
struct tsg_log_instance_t
{
int mode;
unsigned char mode;
int level;
int max_service;
int vsystem_id;
@@ -198,6 +196,7 @@ struct tsg_log_instance_t
int tcp_flow_project_id;
int udp_flow_project_id;
int sum_stat_row_id;
int ipfix_template_interval_pkts;
char tcp_label[MAX_STRING_LEN32];
char udp_label[MAX_STRING_LEN32];
char sasl_username[MAX_STRING_LEN32];
@@ -210,6 +209,7 @@ struct tsg_log_instance_t
id2field_t id2field[LOG_COMMON_MAX];
rd_kafka_t *kafka_handle;
struct topic_stat *service2topic;
struct ipfix_exporter_instance *ipfix_instance;
void *logger;
};

8
test/src/CMakeLists.txt
View File

@@ -23,7 +23,7 @@ add_executable(gtest_action ${PROJECT_SOURCE_DIR}/src/tsg_action.cpp ${PROJECT_S
target_link_libraries(gtest_action gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger maat4 fieldstat3)
add_executable(gtest_sendlog ${PROJECT_SOURCE_DIR}/src/tsg_send_log.cpp ${PROJECT_SOURCE_DIR}/src/tsg_variable.cpp ${PROJECT_SOURCE_DIR}/src/tsg_stat.cpp gtest_common.cpp gtest_kafka.cpp gtest_sendlog.cpp)
target_link_libraries(gtest_sendlog gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger maat4 rdkafka fieldstat3)
target_link_libraries(gtest_sendlog gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger maat4 rdkafka fieldstat3 ipfix_exporter)
set(TSG_MASTER_SRC ${PROJECT_SOURCE_DIR}/src/tsg_entry.cpp
${PROJECT_SOURCE_DIR}/src/tsg_rule.cpp
@@ -47,15 +47,15 @@ set(TSG_MASTER_SRC ${PROJECT_SOURCE_DIR}/src/tsg_entry.cpp
)
add_executable(gtest_master ${TSG_MASTER_SRC} gtest_kafka.cpp gtest_common.cpp gtest_master.cpp)
target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto)
target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter)
add_executable(gtest_sync_state ${PROJECT_SOURCE_DIR}/src/tsg_sync_state.cpp ${PROJECT_SOURCE_DIR}/src/mpack.c ${PROJECT_SOURCE_DIR}/src/tsg_stat.cpp gtest_common.cpp gtest_sync_state.cpp)
target_link_libraries(gtest_sync_state gtest-static cjson ctemplate-static MESA_prof_load MESA_handle_logger fieldstat3)
add_executable(gtest_fieldstat3 ${TSG_MASTER_SRC} gtest_common.cpp gtest_kafka.cpp gtest_fieldstat3.cpp)
target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto)
target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3 crypto ipfix_exporter)
add_executable(gtest_proxy ${TSG_MASTER_SRC} gtest_common.cpp gtest_proxy.cpp)
target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap crypto)
target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap crypto ipfix_exporter)
file(COPY ../pcap/ DESTINATION ./pcap/)