gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature： TSG-21853 Refactoring TFE Kafka infrastructure

Browse Source
This commit is contained in:
luwenpeng
2024-07-19 18:20:04 +08:00
parent 88a7a8c5c4
commit 2045d517ca
25 changed files with 484 additions and 662 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
common/src/tfe_resource.cpp
View File

@@ -1,21 +1,73 @@
#include <MESA/MESA_prof_load.h>
#include <MESA/maat.h>
#include <cjson/cJSON.h>
#include <tfe_kafka_logger.h>
#include <tfe_fieldstat.h>
#include <tfe_proxy.h>
#include <tfe_resource.h>
#include "kafka.h"
#define MAAT_INPUT_JSON 0
#define MAAT_INPUT_REDIS 1
#define MAAT_INPUT_FILE 2
static int scan_table_id[__SCAN_COMMON_TABLE_MAX];
static struct maat *static_maat = NULL;
static tfe_kafka_logger_t *kafka_logger = NULL;
static struct tfe_fieldstat_metric_t *dynamic_fieldstat = NULL;
static char *device_id = NULL;
static char *effective_device_tag=NULL;
static char *device_tag=NULL;
struct tfe_fieldstat_metric_t *fieldstat_handle = NULL;
struct kafka *kafka_handle = NULL;
struct maat *maat_handle = NULL;
static int vsys_id = 0;
static char data_center[1024] = {0};
static char device_group[1024] = {0};
static char device_id[1024] = {0};
static char sled_ip[1024] = {0};
int tfe_get_vsys_id()
{
return vsys_id;
}
const char *tfe_get_device_id()
{
return device_id;
}
const char *tfe_get_data_center()
{
return data_center;
}
const char *tfe_get_device_group()
{
return device_group;
}
const char *tfe_get_device_tag()
{
return device_tag;
}
const char *tfe_get_sled_ip()
{
return sled_ip;
}
struct kafka *tfe_get_kafka_handle()
{
return kafka_handle;
}
struct maat *tfe_get_maat_handle()
{
return maat_handle;
}
struct tfe_fieldstat_metric_t *tfe_get_fieldstat_handle()
{
return fieldstat_handle;
}
static struct tfe_fieldstat_metric_t *create_fieldstat_instance(const char *profile, const char *section, int max_thread, void *logger)
{
@@ -173,129 +225,9 @@ error_out:
return NULL;
}
static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char *section, void *logger)
static char* create_device_tag(const char *profile, const char *section, void *logger)
{
int ret=0, enable=0, vsystem_id=0;
char nic_name[TFE_SYMBOL_MAX] = {0};
char brokerlist[TFE_STRING_MAX] = {0};
char logger_topic[TFE_STRING_MAX] = {0};
char bucket_topic[TFE_STRING_MAX] = {0};
char sasl_username[TFE_STRING_MAX] = {0};
char sasl_passwd[TFE_STRING_MAX] = {0};
tfe_kafka_logger_t *kafka_logger = NULL;
MESA_load_profile_int_def(profile, section, "enable", &enable, 1);
MESA_load_profile_int_def(profile, section, "VSYSTEM_ID", &vsystem_id, 1);
MESA_load_profile_string_def(profile, section, "NIC_NAME", nic_name, sizeof(nic_name), "eth0");
MESA_load_profile_string_def(profile, section, "KAFKA_BROKERLIST", brokerlist, sizeof(brokerlist), "");
MESA_load_profile_string_def(profile, section, "LOGGER_SEND_TOPIC", logger_topic, sizeof(logger_topic), "PROXY-EVENT");
MESA_load_profile_string_def(profile, section, "FILE_BUCKET_TOPIC", bucket_topic, sizeof(bucket_topic), "TRAFFIC-FILE-STREAM-RECORD");
MESA_load_profile_string_def(profile, section, "SASL_USERNAME", sasl_username, sizeof(sasl_username), "");
MESA_load_profile_string_def(profile, section, "SASL_PASSWD", sasl_passwd, sizeof(sasl_passwd), "");
if (!strlen(brokerlist))
{
TFE_LOG_ERROR(logger, "tfe kafka init failed, no brokerlist in profile %s section %s.", profile, section);
return NULL;
}
kafka_logger = tfe_kafka_logger_create(enable, nic_name, brokerlist, logger);
if (kafka_logger == NULL)
{
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create kafka logger.");
return NULL;
}
ret = tfe_logger_create_kafka_topic(kafka_logger, sasl_username, sasl_passwd, logger_topic, TOPIC_LOGGER, logger);
if(ret < 0)
{
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create %s topic.", logger_topic);
return NULL;
}
ret = tfe_logger_create_kafka_topic(kafka_logger, sasl_username, sasl_passwd, bucket_topic, TOPIC_BUCKET, logger);
if(ret < 0)
{
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create %s topic.", bucket_topic);
return NULL;
}
kafka_logger->t_vsys_id=vsystem_id;
TFE_LOG_INFO(logger, "tfe kafka logger : %s", enable ? "ENABLE" : "DISABLE");
TFE_LOG_INFO(logger, "tfe kafka vsystem id : %d", vsystem_id);
TFE_LOG_INFO(logger, "tfe logger kafka topic : %s", logger_topic);
TFE_LOG_INFO(logger, "tfe bucket kafka topic : %s", bucket_topic);
TFE_LOG_INFO(logger, "tfe kafka brokerlist : %s", brokerlist);
if (strlen(sasl_username) > 0 && strlen(sasl_passwd) > 0)
{
TFE_LOG_INFO(logger, "tfe kafka sasl_username : %s", sasl_username);
TFE_LOG_INFO(logger, "tfe kafka sasl_passwd : %s", sasl_passwd);
}
return kafka_logger;
}
static char *cerate_device_id(const char *profile, const char *section, void *logger)
{
int ret = -1;
size_t device_id_size = 0;
char *tsg_sn_file = NULL, *device_id;
const char *device_def_id = "DFT2201925000001";
cJSON *json = NULL, *item = NULL;
char device_id_filepath[TFE_STRING_MAX] = {0};
ret = MESA_load_profile_string_def(profile, section, "device_id_filepath", device_id_filepath, sizeof(device_id_filepath), NULL);
if (ret < 0)
{
TFE_LOG_ERROR(logger, "Invalid device parameter: device_id_filepath not existed in profile %s section %s.", profile, section);
goto finish;
}
tsg_sn_file = tfe_read_file(device_id_filepath, &device_id_size);
if (tsg_sn_file == NULL)
{
TFE_LOG_ERROR(logger, "Invalid device parameter: device sn file not existed.");
goto finish;
}
json = cJSON_Parse(tsg_sn_file);
if (json == NULL)
{
TFE_LOG_ERROR(logger, "Invalid device parameter: %s invalid json format", tsg_sn_file);
goto finish;
}
item = cJSON_GetObjectItem(json, "sn");
if (unlikely(!item || !cJSON_IsString(item)))
{
TFE_LOG_ERROR(logger, "Invalid device parameter: %s invalid json format", tsg_sn_file);
goto finish;
}
device_id = tfe_strdup(item->valuestring);
if(tsg_sn_file)
{
FREE(&tsg_sn_file);
}
cJSON_Delete(json);
TFE_LOG_INFO(logger, "tfe device id : %s", device_id);
return device_id;
finish:
TFE_LOG_INFO(logger, "tfe use default device id : %s", device_def_id);
if (json)
{
cJSON_Delete(json);
}
if(tsg_sn_file)
{
FREE(&tsg_sn_file);
}
return (char *)device_def_id;
}
static char* create_effective_device_tag(const char *profile, const char *section, void *logger)
{
char *effective_device_tag=NULL;
char *c=NULL;
char accept_path[TFE_PATH_MAX] = {0}, accept_tags[TFE_STRING_MAX] = {0};
MESA_load_profile_string_def(profile, section, "accept_path", accept_path, sizeof(accept_path), "");
@@ -307,10 +239,10 @@ static char* create_effective_device_tag(const char *profile, const char *sectio
{
return NULL;
}
effective_device_tag = tfe_strdup(accept_tags);
TFE_LOG_INFO(logger, "tfe device tag : %s", effective_device_tag);
device_tag = tfe_strdup(accept_tags);
TFE_LOG_INFO(logger, "tfe device tag : %s", device_tag);
return effective_device_tag;
return device_tag;
}
void app_dict_table_new_cb(const char *table_name, int table_id, const char* key, const char* table_line, void **ad, long argl, void* argp)
@@ -410,71 +342,62 @@ static int maat_common_table_init()
for (int i = 0; i < __SCAN_COMMON_TABLE_MAX; i++)
{
scan_table_id[i] = maat_get_table_id(static_maat, table_name[i]);
scan_table_id[i] = maat_get_table_id(maat_handle, table_name[i]);
if (scan_table_id[i] < 0)
{
TFE_LOG_ERROR(g_default_logger, "Maat table %s register failed.", table_name[i]);
return -1;
}
}
maat_plugin_table_ex_schema_register(static_maat, "APP_ID_DICT", app_dict_table_new_cb, app_dict_table_free_cb, app_dict_table_dup_cb, 0, NULL);
maat_plugin_table_ex_schema_register(maat_handle, "APP_ID_DICT", app_dict_table_new_cb, app_dict_table_free_cb, app_dict_table_dup_cb, 0, NULL);
return 0;
}
int tfe_bussiness_resouce_init()
int tfe_env_init()
{
const char *profile_path = "./conf/tfe/tfe.conf";
unsigned int thread_num = tfe_proxy_get_work_thread_count();
static_maat = create_maat_feather("static", profile_path, "MAAT", thread_num, g_default_logger);
if (!static_maat)
{
return -1;
}
const char *profile_path = "./conf/tfe/tfe.conf";
kafka_logger = create_kafka_logger(profile_path, "kafka", g_default_logger);
if (!kafka_logger)
{
return -1;
}
MESA_load_profile_int_def(profile_path, "public", "vsys_id", &vsys_id, 0);
MESA_load_profile_string_def(profile_path, "public", "data_center", data_center, sizeof(data_center), "");
MESA_load_profile_string_def(profile_path, "public", "device_group", device_group, sizeof(device_group), "");
MESA_load_profile_string_def(profile_path, "public", "device_id", device_id, sizeof(device_id), "");
dynamic_fieldstat = create_fieldstat_instance(profile_path, "proxy_hits", thread_num, g_default_logger);
if(!dynamic_fieldstat)
{
return -1;
}
char *ptr = getenv("OVERRIDE_SLED_IP");
if (ptr == NULL)
{
return -1;
}
strncpy(sled_ip, ptr, strlen(ptr));
device_id = cerate_device_id(profile_path, "kafka", g_default_logger);
effective_device_tag = create_effective_device_tag(profile_path, "MAAT", g_default_logger);
kafka_handle = kafka_create(profile_path);
if (!kafka_handle)
{
return -1;
}
if (maat_common_table_init())
{
return -1;
}
unsigned int thread_num = tfe_proxy_get_work_thread_count();
maat_handle = create_maat_feather("static", profile_path, "MAAT", thread_num, g_default_logger);
if (!maat_handle)
{
return -1;
}
return 0;
}
dynamic_fieldstat = create_fieldstat_instance(profile_path, "proxy_hits", thread_num, g_default_logger);
if (!dynamic_fieldstat)
{
return -1;
}
void *tfe_bussiness_resouce_get(enum RESOURCE_TYPE type)
{
switch (type)
{
case STATIC_MAAT:
return static_maat;
case KAFKA_LOGGER:
return kafka_logger;
case DEVICE_ID:
return device_id;
case EFFECTIVE_DEVICE_TAG:
return effective_device_tag;
case DYNAMIC_FIELDSTAT:
return dynamic_fieldstat;
default:
return NULL;
}
device_tag = create_device_tag(profile_path, "MAAT", g_default_logger);
if (maat_common_table_init())
{
return -1;
}
return 0;
}
int tfe_bussiness_tableid_get(enum scan_common_table type)
{
return scan_table_id[type];
return scan_table_id[type];
}