#include #include #include #include #include #include #include "kafka.h" #define MAAT_INPUT_JSON 0 #define MAAT_INPUT_REDIS 1 #define MAAT_INPUT_FILE 2 static int scan_table_id[__SCAN_COMMON_TABLE_MAX]; static struct tfe_fieldstat_easy_t *fieldstat4_easy = NULL; static char *device_tag=NULL; struct kafka *kafka_handle = NULL; struct maat *maat_handle = NULL; static int vsys_id = 0; static char data_center[1024] = {0}; static char device_group[1024] = {0}; static char device_id[1024] = {0}; static char sled_ip[1024] = {0}; int tfe_get_vsys_id() { return vsys_id; } const char *tfe_get_device_id() { return device_id; } const char *tfe_get_data_center() { return data_center; } const char *tfe_get_device_group() { return device_group; } const char *tfe_get_device_tag() { return device_tag; } const char *tfe_get_sled_ip() { return sled_ip; } struct kafka *tfe_get_kafka_handle() { return kafka_handle; } struct maat *tfe_get_maat_handle() { return maat_handle; } struct tfe_fieldstat_easy_t *tfe_get_fieldstat_handle() { return fieldstat4_easy; } static tfe_fieldstat_easy_t *create_fieldstat4_instance(const char *profile, const char *section, int max_thread, void *logger) { int cycle=0; char app_name[TFE_STRING_MAX]={0}; char outpath[TFE_STRING_MAX]={0}; int output_fs_interval_ms=0, output_kafka_interval_ms=0; struct tfe_fieldstat_easy_t *fieldstat_easy=NULL; MESA_load_profile_int_def(profile, section, "output_fs_interval_ms", &output_fs_interval_ms, 500); MESA_load_profile_int_def(profile, section, "output_kafka_interval_ms", &output_kafka_interval_ms, 1000); fieldstat_easy = tfe_fieldstat_easy_create(output_kafka_interval_ms); if (fieldstat_easy == NULL) { TFE_LOG_ERROR(logger, "tfe fieldstat init failed, error to create fieldstat metric."); return NULL; } MESA_load_profile_string_def(profile, section, "app_name", app_name, sizeof(app_name), "proxy_rule_hits"); MESA_load_profile_int_def(profile, section, "cycle", &cycle, 0); MESA_load_profile_string_def(profile, section, "outpath", outpath, sizeof(outpath), "metrics/porxy_fieldstat.json"); fieldstat_easy->manipulation = tfe_fieldstat_easy_manipulation_create(app_name, outpath, cycle, max_thread, logger); TFE_LOG_INFO(logger, "tfe fieldstat app_name : %s", app_name); TFE_LOG_INFO(logger, "tfe fieldstat cycle : %d", cycle); TFE_LOG_INFO(logger, "tfe fieldstat outpath : %s", outpath); TFE_LOG_INFO(logger, "tfe output_fs_interval_ms : %d", output_fs_interval_ms); TFE_LOG_INFO(logger, "tfe output_kafka_interval_ms : %d", output_kafka_interval_ms); return fieldstat_easy; } static struct maat *create_maat_feather(const char *instance_name, const char *profile, const char *section, int max_thread, void *logger) { struct maat *target=NULL; int input_mode = 0, maat_stat_on = 0, maat_perf_on = 0; int ret = 0, effect_interval = 60, log_level=0, max_size_mb=0; char table_info[TFE_STRING_MAX] = {0}, inc_cfg_dir[TFE_STRING_MAX] = {0}, ful_cfg_dir[TFE_STRING_MAX] = {0}; char redis_server[TFE_STRING_MAX] = {0}; char redis_port_range[TFE_STRING_MAX] = {0}; char accept_tags[TFE_STRING_MAX] = {0}; char accept_path[TFE_PATH_MAX] = {0}; int redis_port_begin = 0, redis_port_end = 0; int redis_port_select = 0; int redis_db_idx = 0; int deferred_load_on = 0; char json_cfg_file[TFE_STRING_MAX] = {0}, maat_stat_file[TFE_STRING_MAX] = {0}; MESA_load_profile_int_def(profile, section, "maat_input_mode", &(input_mode), 0); MESA_load_profile_int_def(profile, section, "stat_switch", &(maat_stat_on), 1); MESA_load_profile_int_def(profile, section, "perf_switch", &(maat_perf_on), 1); MESA_load_profile_string_def(profile, section, "table_info", table_info, sizeof(table_info), ""); MESA_load_profile_string_def(profile, section, "accept_path", accept_path, sizeof(accept_path), ""); MESA_load_profile_string_def(profile, section, "json_cfg_file", json_cfg_file, sizeof(json_cfg_file), ""); MESA_load_profile_string_def(profile, section, "maat_redis_server", redis_server, sizeof(redis_server), ""); MESA_load_profile_string_def(profile, section, "maat_redis_port_range", redis_port_range, sizeof(redis_server), "6379"); MESA_load_profile_int_def(profile, section, "maat_redis_db_index", &(redis_db_idx), 0); MESA_load_profile_string_def(profile, section, "inc_cfg_dir", inc_cfg_dir, sizeof(inc_cfg_dir), ""); MESA_load_profile_string_def(profile, section, "full_cfg_dir", ful_cfg_dir, sizeof(ful_cfg_dir), ""); MESA_load_profile_string_def(profile, section, "stat_file", maat_stat_file, sizeof(maat_stat_file), ""); MESA_load_profile_int_def(profile, section, "effect_interval_s", &(effect_interval), 60); MESA_load_profile_int_def(profile, section, "deferred_load_on", &(deferred_load_on), 0); MESA_load_profile_int_def(profile, section, "log_level", &(log_level), LOG_LEVEL_FATAL); MESA_load_profile_int_def(profile, section, "log_max_size_mb", &(max_size_mb), 0); effect_interval *= 1000; //convert s to ms struct maat_options *opts = maat_options_new(); maat_options_set_logger(opts, "log/maat.log", (enum log_level)log_level); //maat_options_set_log_file_max_size(opts, max_size_mb); maat_options_set_instance_name(opts, instance_name); maat_options_set_caller_thread_number(opts, max_thread); switch (input_mode) { case MAAT_INPUT_JSON: if (!strlen(json_cfg_file)) { TFE_LOG_ERROR(logger, "Invalid json_cfg_file, MAAT init failed."); goto error_out; } maat_options_set_json_file(opts, json_cfg_file); break; case MAAT_INPUT_REDIS: if (!strlen(redis_server)) { TFE_LOG_ERROR(logger, "Invalid maat_redis_server, MAAT init failed."); goto error_out; } ret = sscanf(redis_port_range, "%d-%d", &redis_port_begin, &redis_port_end); if (ret == 1) { redis_port_select = redis_port_begin; } else if (ret == 2) { srand(time(NULL)); redis_port_select = redis_port_begin + rand() % (redis_port_end - redis_port_begin); } else { TFE_LOG_ERROR(logger, "Invalid redis port range %s, MAAT init failed.", redis_port_range); goto error_out; } maat_options_set_redis(opts, redis_server, redis_port_select, redis_db_idx); break; case MAAT_INPUT_FILE: if (!strlen(ful_cfg_dir)) { TFE_LOG_ERROR(logger, "Invalid ful_cfg_dir, MAAT init failed."); goto error_out; } if (!strlen(inc_cfg_dir)) { TFE_LOG_ERROR(logger, "Invalid inc_cfg_dir, MAAT init failed."); goto error_out; } maat_options_set_iris(opts, ful_cfg_dir, inc_cfg_dir); break; default: TFE_LOG_ERROR(logger, "Invalid MAAT Input Mode: %d.", input_mode); goto error_out; break; } maat_options_set_foreign_cont_dir(opts, "./pangu_files"); if (maat_stat_on) { maat_options_set_stat_on(opts); maat_options_set_stat_file(opts, maat_stat_file); if (maat_perf_on) { maat_options_set_perf_on(opts); } } if (deferred_load_on) { maat_options_set_deferred_load_on(opts); } if (strlen(accept_path) > 0) { MESA_load_profile_string_def(accept_path, "maat", "ACCEPT_TAGS", accept_tags, sizeof(accept_tags), "{\"tags\":[{\"tag\":\"device_id\",\"value\":\"device_1\"}]}"); maat_options_set_accept_tags(opts, accept_tags); TFE_LOG_INFO(logger, "tfe accept tags : %s", accept_tags); } target = maat_new(opts, table_info); if (!target) { TFE_LOG_ERROR(logger, "%s MAAT init failed.", __FUNCTION__); goto error_out; } maat_options_free(opts); return target; error_out: maat_options_free(opts); return NULL; } static char* create_device_tag(const char *profile, const char *section, void *logger) { char accept_path[TFE_PATH_MAX] = {0}, accept_tags[TFE_STRING_MAX] = {0}; MESA_load_profile_string_def(profile, section, "accept_path", accept_path, sizeof(accept_path), ""); if(strlen(accept_path) > 0) { MESA_load_profile_string_def(accept_path, "maat", "ACCEPT_TAGS", accept_tags, sizeof(accept_tags), ""); } if(strlen(accept_tags) <= 0) { return NULL; } device_tag = tfe_strdup(accept_tags); TFE_LOG_INFO(logger, "tfe device tag : %s", device_tag); return device_tag; } void app_dict_table_new_cb(const char *table_name, int table_id, const char* key, const char* table_line, void **ad, long argl, void* argp) { int ret=0; size_t offset=0, len=0; char *app_id_str=NULL, *group_id_str=NULL; struct app_id_dict *app_dict=ALLOC(struct app_id_dict, 1); ret = maat_helper_read_column(table_line, 1, &offset, &len); if(ret >= 0) { app_id_str=ALLOC(char, len+1); memcpy(app_id_str, table_line+offset, len); app_dict->app_id=atoi(app_id_str); FREE(&app_id_str); } ret = maat_helper_read_column(table_line, 18, &offset, &len); if(ret >= 0) { group_id_str=ALLOC(char, len+1); memcpy(group_id_str, table_line+offset, len); app_dict->group_id=atoll(group_id_str); FREE(&group_id_str); } app_dict->ref_cnt=1; pthread_mutex_init(&(app_dict->lock), NULL); *ad=app_dict; return; } void app_dict_table_free_cb(int table_id, void **ad, long argl, void* argp) { if(*ad==NULL) { return; } struct app_id_dict *app_dict=(struct app_id_dict *)(*ad); pthread_mutex_lock(&(app_dict->lock)); app_dict->ref_cnt--; if(app_dict->ref_cnt>0) { pthread_mutex_unlock(&(app_dict->lock)); return; } pthread_mutex_unlock(&(app_dict->lock)); pthread_mutex_destroy(&(app_dict->lock)); FREE(&app_dict); *ad=NULL; return; } void app_id_dict_free(struct app_id_dict *app_dict) { app_dict_table_free_cb(0, (void **)&app_dict, 0, NULL); } void app_dict_table_dup_cb(int table_id, void **to, void **from, long argl, void* argp) { struct app_id_dict *app_dict=(struct app_id_dict *)(*from); pthread_mutex_lock(&(app_dict->lock)); app_dict->ref_cnt++; pthread_mutex_unlock(&(app_dict->lock)); *to=app_dict; return; } int get_category_type_str2idx(const char *category) { size_t i = 0; const char *category_name[] = {"unknown", "geoip_city", "geoip_country", "geoip_asn", "website_classfication", "internet_service", "security_threat", "compliance_risk"}; for (i = 0; i < sizeof(category_name) / sizeof(const char *); i++) { if (0 == strcasecmp(category, category_name[i])) break; } return i; } void library_tag_new_cb(const char *table_name, int table_id, const char* key, const char* table_line, void **ad, long argl, void* argp) { int ret=0; size_t offset=0, len=0; char category[256]={0}; struct library_tag_ctx *library_tags = ALLOC(struct library_tag_ctx, 1); ret = maat_helper_read_column(table_line, 1, &offset, &len); if(ret >= 0) { char *tag_id_str=ALLOC(char, len+1); memcpy(tag_id_str, table_line+offset, len); library_tags->tag_id=atoi(tag_id_str); FREE(&tag_id_str); } ret = maat_helper_read_column(table_line, 3, &offset, &len); if(ret >= 0) { memcpy(category, table_line+offset, len); library_tags->category=(enum category_type)get_category_type_str2idx(category); } ret = maat_helper_read_column(table_line, 4, &offset, &len); if(ret >= 0) { library_tags->tag_key=ALLOC(char, len+1); memcpy(library_tags->tag_key, table_line+offset, len); } ret = maat_helper_read_column(table_line, 5, &offset, &len); if(ret >= 0) { library_tags->tag_value=ALLOC(char, len+1); memcpy(library_tags->tag_value, table_line+offset, len); } library_tags->ref_cnt=1; pthread_mutex_init(&(library_tags->lock), NULL); *ad=library_tags; return; } void library_tag_free_cb(int table_id, void **ad, long argl, void* argp) { if(*ad==NULL) { return; } struct library_tag_ctx *library_tags=(struct library_tag_ctx *)(*ad); pthread_mutex_lock(&(library_tags->lock)); library_tags->ref_cnt--; if(library_tags->ref_cnt>0) { pthread_mutex_unlock(&(library_tags->lock)); return; } pthread_mutex_unlock(&(library_tags->lock)); pthread_mutex_destroy(&(library_tags->lock)); if(library_tags->tag_key) { FREE(&library_tags->tag_key); } if(library_tags->tag_value) { FREE(&library_tags->tag_value); } FREE(&library_tags); *ad=NULL; return; } void library_tag_dup_cb(int table_id, void **to, void **from, long argl, void* argp) { struct library_tag_ctx *library_tags=(struct library_tag_ctx *)(*from); pthread_mutex_lock(&(library_tags->lock)); library_tags->ref_cnt++; pthread_mutex_unlock(&(library_tags->lock)); *to=library_tags; return; } void library_tag_free(struct library_tag_ctx *library_tags) { library_tag_free_cb(0, (void **)&library_tags, 0, NULL); } static int maat_common_table_init() { const char * table_name[__SCAN_COMMON_TABLE_MAX]; table_name[PXY_CTRL_SOURCE_IP] = "ATTR_SOURCE_IP"; table_name[PXY_CTRL_DESTINATION_IP]="ATTR_DESTINATION_IP"; table_name[PXY_CTRL_INTERNAL_IP] = "ATTR_INTERNAL_IP"; table_name[PXY_CTRL_EXTERNAL_IP] = "ATTR_EXTERNAL_IP"; table_name[PXY_CTRL_SOURCE_PORT] = "ATTR_SOURCE_PORT"; table_name[PXY_CTRL_DESTINATION_PORT] = "ATTR_DESTINATION_PORT"; table_name[PXY_CTRL_INTERNAL_PORT] = "ATTR_INTERNAL_PORT"; table_name[PXY_CTRL_EXTERNAL_PORT] = "ATTR_EXTERNAL_PORT"; table_name[PXY_CTRL_IP_PROTOCOL] = "ATTR_IP_PROTOCOL"; table_name[PXY_CTRL_SUBSCRIBER_ID] = "ATTR_SUBSCRIBER_ID"; table_name[PXY_CTRL_APP_ID_DICT] = "APP_ID_DICT"; table_name[PXY_CTRL_LIBRARY_TAG] = "LIBRARY_TAG"; table_name[PXY_CTRL_IMSI]="ATTR_GTP_IMSI"; table_name[PXY_CTRL_APN]="ATTR_GTP_APN"; table_name[PXY_CTRL_PHONE_NUMBER]="ATTR_GTP_PHONE_NUMBER"; table_name[PXY_CTRL_GTP_IMEI]="ATTR_GTP_IMEI"; for (int i = 0; i < __SCAN_COMMON_TABLE_MAX; i++) { scan_table_id[i] = maat_get_table_id(maat_handle, table_name[i]); if (scan_table_id[i] < 0) { TFE_LOG_ERROR(g_default_logger, "Maat table %s register failed.", table_name[i]); return -1; } } maat_plugin_table_ex_schema_register(maat_handle, "APP_ID_DICT", app_dict_table_new_cb, app_dict_table_free_cb, app_dict_table_dup_cb, 0, NULL); maat_plugin_table_ex_schema_register(maat_handle, "LIBRARY_TAG", library_tag_new_cb, library_tag_free_cb, library_tag_dup_cb, 0, NULL); return 0; } int tfe_env_init() { const char *profile_path = "./conf/tfe/tfe.conf"; MESA_load_profile_int_def(profile_path, "public", "vsys_id", &vsys_id, 0); MESA_load_profile_string_def(profile_path, "public", "data_center", data_center, sizeof(data_center), ""); MESA_load_profile_string_def(profile_path, "public", "device_group", device_group, sizeof(device_group), ""); MESA_load_profile_string_def(profile_path, "public", "device_id", device_id, sizeof(device_id), ""); char *ptr = getenv("OVERRIDE_SLED_IP"); if (ptr == NULL) { return -1; } strncpy(sled_ip, ptr, strlen(ptr)); kafka_handle = kafka_create(profile_path); if (!kafka_handle) { return -1; } unsigned int thread_num = tfe_proxy_get_work_thread_count(); maat_handle = create_maat_feather("static", profile_path, "MAAT", thread_num, g_default_logger); if (!maat_handle) { return -1; } fieldstat4_easy = create_fieldstat4_instance(profile_path, "proxy_hits", thread_num, g_default_logger); if(!fieldstat4_easy) { return -1; } device_tag = create_device_tag(profile_path, "MAAT", g_default_logger); if (maat_common_table_init()) { return -1; } return 0; } int tfe_bussiness_tableid_get(enum scan_common_table type) { return scan_table_id[type]; }