From 03d05dd73ec842cd05a4f29f6d2ff07f974bc9ad Mon Sep 17 00:00:00 2001 From: luwenpeng Date: Wed, 24 Jun 2020 16:40:53 +0800 Subject: [PATCH] =?UTF-8?q?TSG-1531=20tfe=20=E4=BB=A3=E7=A0=81=E6=95=B4?= =?UTF-8?q?=E7=90=86,=20=E5=B0=86=E5=A4=9A=E6=8F=92=E4=BB=B6=E5=85=AC?= =?UTF-8?q?=E7=94=A8=E7=9A=84=E5=9F=BA=E7=A1=80=E4=BB=A3=E7=A0=81=E7=A7=BB?= =?UTF-8?q?=E5=8A=A8=E5=88=B0=20tfe=20init=20=E9=98=B6=E6=AE=B5=20=091.?= =?UTF-8?q?=E5=B0=86=20kafka=20=E7=9A=84=E5=88=9D=E5=A7=8B=E5=8C=96?= =?UTF-8?q?=E4=BB=8E=20pangu=20init=20=E9=98=B6=E6=AE=B5=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E5=88=B0=20tfe=20init=20=E9=98=B6=E6=AE=B5=20=092.=E5=B0=86=20?= =?UTF-8?q?device=20id=20=E7=9A=84=E8=8E=B7=E5=8F=96=E4=BB=8E=20pangu=20in?= =?UTF-8?q?it=20=E9=98=B6=E6=AE=B5=E7=A7=BB=E5=8A=A8=E5=88=B0=20tfe=20init?= =?UTF-8?q?=20=E9=98=B6=E6=AE=B5=20=093.=E5=B0=86=20kafka=20=E7=9A=84?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E9=A1=B9=E4=BB=8E=20pangu.conf=20=E7=A7=BB?= =?UTF-8?q?=E5=8A=A8=E5=88=B0=20tfe.conf=20=094.=E5=B0=86=20maat=20=20?= =?UTF-8?q?=E7=9A=84=E9=85=8D=E7=BD=AE=E9=A1=B9=E4=BB=8E=20pangu.conf=20?= =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E5=88=B0=20tfe.conf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/CMakeLists.txt | 2 +- common/include/tfe_resource.h | 2 + common/src/tfe_resource.cpp | 109 ++++++++++++++++-- conf/doh/doh.conf | 13 +-- conf/pangu/pangu_pxy.conf | 26 +---- conf/tfe/tfe.conf | 51 ++++++++ plugin/business/doh/src/doh.cpp | 2 +- plugin/business/doh/src/logger.cpp | 66 +---------- .../business/pangu-http/src/pangu_logger.cpp | 96 ++------------- plugin/business/traffic-mirror/src/entry.cpp | 2 +- resource/pangu/table_info.conf | 46 ++++++++ 11 files changed, 223 insertions(+), 192 deletions(-) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index d65f52e..5e6708b 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -1,7 +1,7 @@ add_library(common src/tfe_utils.cpp src/tfe_types.cpp src/tfe_future.cpp src/tfe_http.cpp src/tfe_plugin.cpp src/tfe_rpc.cpp src/tfe_cmsg.cpp src/tfe_kafka_logger.cpp src/tfe_resource.cpp) target_include_directories(common PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include) target_link_libraries(common PUBLIC libevent-static libevent-static-openssl libevent-static-pthreads) -target_link_libraries(common PUBLIC MESA_handle_logger) +target_link_libraries(common PUBLIC MESA_handle_logger cjson) ### UNITTEST CASE add_executable(test-addr test/test_addr.cpp src/tfe_types.cpp src/tfe_utils.cpp) diff --git a/common/include/tfe_resource.h b/common/include/tfe_resource.h index 7a70b0e..7ec59f1 100644 --- a/common/include/tfe_resource.h +++ b/common/include/tfe_resource.h @@ -4,6 +4,8 @@ enum RESOURCE_TYPE { STATIC_MAAT, DYNAMINC_MAAT, + KAFKA_LOGGER, + DEVICE_ID, }; int tfe_bussiness_resouce_init(); diff --git a/common/src/tfe_resource.cpp b/common/src/tfe_resource.cpp index 92a8efb..646bd18 100644 --- a/common/src/tfe_resource.cpp +++ b/common/src/tfe_resource.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include @@ -10,6 +12,8 @@ static Maat_feather_t static_maat = NULL; static Maat_feather_t dynamic_maat = NULL; +static tfe_kafka_logger_t *kafka_logger = NULL; +static char *device_id = NULL; static Maat_feather_t create_maat_feather(const char *instance_name, const char *profile, const char *section, int max_thread, void *logger) { @@ -135,9 +139,89 @@ error_out: return NULL; } +static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char *section, void *logger) +{ + int enable = 0; + char nic_name[64] = {0}; + char brokerlist[TFE_STRING_MAX] = {0}; + char topic_name[TFE_STRING_MAX] = {0}; + tfe_kafka_logger_t *kafka_logger = NULL; + + MESA_load_profile_int_def(profile, section, "enable", &enable, 1); + MESA_load_profile_string_def(profile, section, "NIC_NAME", nic_name, sizeof(nic_name), "eth0"); + MESA_load_profile_string_def(profile, section, "KAFKA_BROKERLIST", brokerlist, sizeof(brokerlist), ""); + MESA_load_profile_string_def(profile, section, "KAFKA_TOPIC", topic_name, sizeof(topic_name), "POLICY-EVENT-LOG"); + + if (!strlen(brokerlist)) + { + TFE_LOG_ERROR(logger, "tfe kafka init failed, no brokerlist in profile %s section %s.", profile, section); + return NULL; + } + + kafka_logger = tfe_kafka_logger_create(enable, nic_name, brokerlist, topic_name, logger); + if (kafka_logger == NULL) + { + TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create kafka logger."); + return NULL; + } + + TFE_LOG_INFO(logger, "tfe kafka logger : %s", enable ? "ENABLE" : "DISABLE"); + TFE_LOG_INFO(logger, "tfe kafka topic : %s", topic_name); + TFE_LOG_INFO(logger, "tfe kafka brokerlist : %s", brokerlist); + + return kafka_logger; +} + +static char *cerate_device_id(const char *profile, const char *section, void *logger) +{ + int ret = -1; + size_t device_id_size = 0; + char *tsg_sn_file = NULL, *device_id; + + const char *device_def_id = "DFT2201925000001"; + cJSON *json = NULL, *item = NULL; + char device_id_filepath[TFE_STRING_MAX] = {0}; + + ret = MESA_load_profile_string_def(profile, section, "device_id_filepath", device_id_filepath, sizeof(device_id_filepath), NULL); + if (ret < 0) + { + TFE_LOG_ERROR(logger, "Invalid device parameter: device_id_filepath not existed in profile %s section %s.", profile, section); + goto finish; + } + tsg_sn_file = tfe_read_file(device_id_filepath, &device_id_size); + if (tsg_sn_file == NULL) + { + TFE_LOG_ERROR(logger, "Invalid device parameter: device sn file not existed."); + goto finish; + } + json = cJSON_Parse(tsg_sn_file); + if (json == NULL) + { + TFE_LOG_ERROR(logger, "Invalid device parameter: %s invalid json format", tsg_sn_file); + goto finish; + } + item = cJSON_GetObjectItem(json, "sn"); + if (unlikely(!item || !cJSON_IsString(item))) + { + TFE_LOG_ERROR(logger, "Invalid device parameter: %s invalid json format", tsg_sn_file); + goto finish; + } + device_id = tfe_strdup(item->valuestring); + + cJSON_Delete(json); + TFE_LOG_INFO(logger, "tfe device id : %s", device_id); + + return device_id; +finish: + TFE_LOG_INFO(logger, "tfe use default device id : %s", device_def_id); + if (json) + cJSON_Delete(json); + return (char *)device_def_id; +} + int tfe_bussiness_resouce_init() { - const char *profile_path = "./conf/pangu/pangu_pxy.conf"; + const char *profile_path = "./conf/tfe/tfe.conf"; unsigned int thread_num = tfe_proxy_get_work_thread_count(); static_maat = create_maat_feather("static", profile_path, "MAAT", thread_num, g_default_logger); if (!static_maat) @@ -151,19 +235,30 @@ int tfe_bussiness_resouce_init() return -1; } + kafka_logger = create_kafka_logger(profile_path, "kafka", g_default_logger); + if (!kafka_logger) + { + return -1; + } + + device_id = cerate_device_id(profile_path, "kafka", g_default_logger); + return 0; } void *tfe_bussiness_resouce_get(enum RESOURCE_TYPE type) { - if (type == STATIC_MAAT) + switch (type) { + case STATIC_MAAT: return static_maat; - } - if (type == DYNAMINC_MAAT) - { + case DYNAMINC_MAAT: return dynamic_maat; + case KAFKA_LOGGER: + return kafka_logger; + case DEVICE_ID: + return device_id; + default: + return NULL; } - - return NULL; } \ No newline at end of file diff --git a/conf/doh/doh.conf b/conf/doh/doh.conf index 0da64de..b128f71 100644 --- a/conf/doh/doh.conf +++ b/conf/doh/doh.conf @@ -17,18 +17,11 @@ table_addr=TSG_SECURITY_ADDR # default TSG_FIELD_DOH_QNAME table_qname=TSG_FIELD_DOH_QNAME # default TSG_FIELD_HTTP_HOST -table_host=TSG_FIELD_HTTP_HOST +table_host=TSG_FIELD_DOH_HOST [kafka] -# default NULL -device_id_filepath==/opt/tsg/etc/tsg_sn.json # default 0 ENTRANCE_ID=0 # default 1 -en_sendlog=1 -# default eth0 -NIC_NAME=eth0 -# defautl empty -kafka_brokerlist=192.168.40.224:9092 -# default POLICY-DOH-LOG -kafka_topic=POLICY-DOH-LOG \ No newline at end of file +# if enable "en_sendlog", the iterm "tfe.conf [kafka] enable" must set 1 +en_sendlog=1 \ No newline at end of file diff --git a/conf/pangu/pangu_pxy.conf b/conf/pangu/pangu_pxy.conf index 088b1fe..73d2eec 100644 --- a/conf/pangu/pangu_pxy.conf +++ b/conf/pangu/pangu_pxy.conf @@ -2,11 +2,9 @@ log_level=10 [log] -nic_name=eth4 entrance_id=0 -device_id_filepath=/opt/tsg/etc/tsg_sn.json -kafka_brokerlist=10.4.34.10:9092,10.4.34.11:9092,10.4.34.12:9092,10.4.34.13:9092,10.4.34.14:9092,10.4.34.15:9092,10.4.34.16:9092,10.4.34.17:9092,10.4.34.18:9092,10.4.34.19:9092 -kafka_topic=policy-event-log +# default 1, if enable "en_sendlog", the iterm "tfe.conf [kafka] enable" must set 1 +en_sendlog=1 #Addresses of minio. Format is defined by WiredLB. minio_ip_list=10.4.35.42-46; minio_listen_port=9000 @@ -57,26 +55,6 @@ redis_server=192.168.40.137 redis_port=6379 redis_db_index=5 -[maat] -# 0:json 1: redis 2: iris -maat_input_mode=1 -table_info=resource/pangu/table_info.conf -json_cfg_file=resource/ -stat_file=log/pangu_scan.fs2 -full_cfg_dir=pangu_policy/full/index/ -inc_cfg_dir=pangu_policy/inc/index/ -maat_redis_server=10.4.34.4 -maat_redis_port_range=6380-6389 -maat_redis_db_index=4 -effect_interval_s=1 -accept_tags={"tags":[{"tag":"location","value":"Astana"}]} -[dynamic_maat] -maat_input_mode=1 -table_info=resource/pangu/dynamic_maat_table_info.conf -maat_redis_server=10.4.20.151 -maat_redis_port_range=6380-6389 -maat_redis_db_index=0 -effect_interval_s=1 [tango_cache] enable_cache=1 min_cache_obj_size=512 diff --git a/conf/tfe/tfe.conf b/conf/tfe/tfe.conf index 1ca97ea..ac92c87 100644 --- a/conf/tfe/tfe.conf +++ b/conf/tfe/tfe.conf @@ -81,3 +81,54 @@ loglevel=20 [traffic_mirror] device=eth4 type=1 + +[kafka] +enable=1 +NIC_NAME=enp2s0 +kafka_brokerlist=192.168.40.224:9092 +kafka_topic=POLICY-EVENT-LOG +device_id_filepath==/opt/tsg/etc/tsg_sn.json + +[maat] +# 0:json 1:redis 2:iris +maat_input_mode=1 +stat_switch=1 +perf_switch=1 +table_info=resource/pangu/table_info.conf +accept_tags={"tags":[{"tag":"location","value":"Astana"}]} +stat_file=log/pangu_scan.fs2 +effect_interval_s=1 + +# json mode conf iterm +json_cfg_file=resource/pangu/pangu_http.json + +# redis mode conf iterm +maat_redis_server=10.4.34.4 +maat_redis_port_range=6380-6389 +maat_redis_db_index=4 + +# iris mode conf iterm +full_cfg_dir=pangu_policy/full/index/ +inc_cfg_dir=pangu_policy/inc/index/ + +[dynamic_maat] +# 0:json 1:redis 2:iris +maat_input_mode=1 +stat_switch=1 +perf_switch=1 +table_info=resource/pangu/dynamic_maat_table_info.conf +accept_tags={"tags":[{"tag":"location","value":"Astana"}]} +stat_file=log/pangu_scan.fs2 +effect_interval_s=1 + +# json mode conf iterm +json_cfg_file=resource/pangu/pangu_http.json + +# redis mode conf iterm +maat_redis_server=10.4.34.4 +maat_redis_port_range=6380-6389 +maat_redis_db_index=4 + +# redis mode conf iterm +full_cfg_dir=pangu_policy/full/index/ +inc_cfg_dir=pangu_policy/inc/index/ \ No newline at end of file diff --git a/plugin/business/doh/src/doh.cpp b/plugin/business/doh/src/doh.cpp index 1bb9e02..adc1d0c 100644 --- a/plugin/business/doh/src/doh.cpp +++ b/plugin/business/doh/src/doh.cpp @@ -758,4 +758,4 @@ struct tfe_plugin doh_spec = { .on_session_begin = doh_on_begin, .on_session_data = doh_on_data, .on_session_end = doh_on_end}; -TFE_PLUGIN_REGISTER(doh, doh_spec) \ No newline at end of file +TFE_PLUGIN_REGISTER(DOH, doh_spec) \ No newline at end of file diff --git a/plugin/business/doh/src/logger.cpp b/plugin/business/doh/src/logger.cpp index 91762b3..e2f0e76 100644 --- a/plugin/business/doh/src/logger.cpp +++ b/plugin/business/doh/src/logger.cpp @@ -276,80 +276,22 @@ static void add_dns_info_to_log(cJSON *common_obj, dns_info_t *dns_info) cJSON_AddNumberToObject(common_obj, "doh_sub", dns_sec); } -static const char *tfe_device_id_create(const char *profile, const char *section, void *local_logger) -{ - int ret = -1; - size_t device_id_size = 0; - char *tsg_sn_file = NULL, *device_id; - const char *device_def_id = "DFT2201925000001"; - cJSON *json = NULL, *item = NULL; - char device_id_filepath[TFE_STRING_MAX] = {0}; - - ret = MESA_load_profile_string_def(profile, section, "device_id_filepath", device_id_filepath, sizeof(device_id_filepath), NULL); - if (ret < 0) - { - TFE_LOG_ERROR(local_logger, "Doh log init failed, no device_path in profile %s section %s.", profile, section); - goto finish; - } - tsg_sn_file = tfe_read_file(device_id_filepath, &device_id_size); - if (tsg_sn_file == NULL) - { - TFE_LOG_ERROR(local_logger, "Doh log init failed, %s not existed.", tsg_sn_file); - goto finish; - } - json = cJSON_Parse(tsg_sn_file); - if (json == NULL) - { - TFE_LOG_ERROR(local_logger, "invalid device parameter: file = %s", tsg_sn_file); - goto finish; - } - item = cJSON_GetObjectItem(json, "sn"); - if (unlikely(!item || !cJSON_IsString(item))) - { - TFE_LOG_ERROR(local_logger, "Invalid device parameter: %s invalid json format", tsg_sn_file); - } - device_id = tfe_strdup(item->valuestring); - - cJSON_Delete(json); - return device_id; -finish: - return device_def_id; -} - int doh_kafka_init(const char *profile, struct doh_conf *conf) { - char nic_name[64] = {0}; - char brokerlist[TFE_STRING_MAX] = {0}; - char topic_name[TFE_STRING_MAX] = {0}; const char *section = "kafka"; - MESA_load_profile_int_def(profile, section, "ENTRANCE_ID", &(conf->entry_id), 0); MESA_load_profile_int_def(profile, section, "en_sendlog", &conf->en_sendlog, 1); - MESA_load_profile_string_def(profile, section, "NIC_NAME", nic_name, sizeof(nic_name), "eth0"); - MESA_load_profile_string_def(profile, section, "KAFKA_BROKERLIST", brokerlist, sizeof(brokerlist), ""); - MESA_load_profile_string_def(profile, section, "KAFKA_TOPIC", topic_name, sizeof(topic_name), "POLICY-DOH-LOG"); - - TFE_LOG_INFO(conf->local_logger, "Doh sendlog : %s", conf->en_sendlog ? "ENABLE" : "DISABLE"); if (!conf->en_sendlog) { return 0; } - conf->device_id = tfe_device_id_create(profile, section, conf->local_logger); - if (!strlen(brokerlist)) + conf->device_id = (const char *)tfe_bussiness_resouce_get(DEVICE_ID); + conf->kafka_logger = (tfe_kafka_logger_t *)tfe_bussiness_resouce_get(KAFKA_LOGGER); + if (conf->kafka_logger && !conf->kafka_logger->enable) { - TFE_LOG_ERROR(conf->local_logger, "Doh log init failed, no brokerlist in profile %s section %s.", profile, section); + TFE_LOG_ERROR(conf->local_logger, "Doh sendlog ENABLE, but tfe kafka logger DISABLED."); return -1; } - conf->kafka_logger = tfe_kafka_logger_create(conf->en_sendlog, nic_name, brokerlist, topic_name, conf->local_logger); - if (conf->kafka_logger == NULL) - { - TFE_LOG_ERROR(conf->local_logger, "Doh kafka init failed, error to create kafka logger."); - return -1; - } - - TFE_LOG_INFO(conf->local_logger, "Doh device id : %s", conf->device_id); - TFE_LOG_INFO(conf->local_logger, "Doh kafka topic : %s", topic_name); - TFE_LOG_INFO(conf->local_logger, "Doh kafka brokerlist : %s", brokerlist); return 0; } diff --git a/plugin/business/pangu-http/src/pangu_logger.cpp b/plugin/business/pangu-http/src/pangu_logger.cpp index 61ccebf..c7487b5 100644 --- a/plugin/business/pangu-http/src/pangu_logger.cpp +++ b/plugin/business/pangu-http/src/pangu_logger.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "pangu_logger.h" @@ -14,13 +15,8 @@ struct json_spec struct pangu_logger { int entry_id; - unsigned int en_sendlog; - unsigned int en_sendlog_meta; - unsigned int en_sendlog_body; - const char *device_id; - void* global_logger; void* local_logger; unsigned long long send_cnt; @@ -45,101 +41,29 @@ enum _log_action //Bigger action number is prior. __LG_ACTION_MAX }; -static const char* tfe_device_id_create(const char* profile, const char* section, void* local_logger) -{ - int ret=-1; - size_t device_id_size=0; - char *tsg_sn_file=NULL, *device_id; - const char *device_def_id = "DFT2201925000001"; - cJSON *json=NULL,*item=NULL; - char device_id_filepath[TFE_STRING_MAX] = { 0 }; - - ret = MESA_load_profile_string_def(profile, section, "device_id_filepath",device_id_filepath,sizeof(device_id_filepath), NULL); - if(ret<0) - { - TFE_LOG_ERROR(local_logger,"Pangu log init failed, no device_path in profile %s section %s.", profile, section); - goto finish; - } - tsg_sn_file = tfe_read_file(device_id_filepath, &device_id_size); - if(tsg_sn_file==NULL) - { - TFE_LOG_ERROR(local_logger,"Pangu log init failed, %s not existed.", tsg_sn_file); - goto finish; - } - json=cJSON_Parse(tsg_sn_file); - if(json==NULL) - { - TFE_LOG_ERROR(local_logger, "invalid device parameter: file = %s", tsg_sn_file); - goto finish; - } - item=cJSON_GetObjectItem(json, "sn"); - if(unlikely(!item || !cJSON_IsString(item))) - { - TFE_LOG_ERROR(local_logger, "Invalid device parameter: %s invalid json format", tsg_sn_file); - } - device_id = tfe_strdup(item->valuestring); - - cJSON_Delete(json); - return device_id; -finish: - return device_def_id; -} - struct pangu_logger* pangu_log_handle_create(const char* profile, const char* section, void* local_logger) { - int ret=-1; - char nic_name[64]={0}; - char brokerlist[TFE_STRING_MAX] = { 0 }; - char topic_name[TFE_STRING_MAX] = { 0 }; struct tango_cache_parameter *log_file_upload_para=NULL; - struct pangu_logger* instance=ALLOC(struct pangu_logger,1); instance->local_logger=local_logger; TFE_LOG_INFO(local_logger,"Pangu log is inititating from %s section %s.", profile, section); - + MESA_load_profile_int_def(profile, section, "ENTRANCE_ID",&(instance->entry_id),0); MESA_load_profile_uint_def(profile, section, "en_sendlog", &instance->en_sendlog, 1); - MESA_load_profile_uint_def(profile, section, "en_sendlog_meta", &instance->en_sendlog_meta, 1); - MESA_load_profile_uint_def(profile, section, "en_sendlog_body", &instance->en_sendlog_body, 1); - - if (!instance->en_sendlog) - { - instance->en_sendlog_body = 0; - instance->en_sendlog_meta = 0; - } - - TFE_LOG_INFO(local_logger, "Pangu sendlog : %s", instance->en_sendlog ? "ENABLE" : "DISABLE"); - TFE_LOG_INFO(local_logger, "Pangu sendlog meta : %s", instance->en_sendlog_meta ? "ENABLE" : "DISABLE"); - TFE_LOG_INFO(local_logger, "Pangu sendlog body : %s", instance->en_sendlog_body ? "ENABLE" : "DISABLE"); + TFE_LOG_INFO(local_logger, "Pangu sendlog : %s", instance->en_sendlog ? "ENABLE" : "DISABLE"); if (!instance->en_sendlog) { return instance; } - MESA_load_profile_string_def(profile, section, "NIC_NAME",nic_name,sizeof(nic_name),"eth0"); - MESA_load_profile_int_def(profile, section, "ENTRANCE_ID",&(instance->entry_id),0); - - instance->device_id = tfe_device_id_create(profile, section, local_logger); - TFE_LOG_INFO(local_logger, "Pangu device id : %s", instance->device_id); - - ret=MESA_load_profile_string_def(profile, section,"KAFKA_BROKERLIST", brokerlist, sizeof(brokerlist), NULL); - if(ret<0) - { - TFE_LOG_ERROR(local_logger,"Pangu log init failed, no brokerlist in profile %s section %s.", profile, section); - goto error_out; - } - MESA_load_profile_string_def(profile, section,"KAFKA_TOPIC", topic_name, sizeof(topic_name), "POLICY-EVENT-LOG"); - - TFE_LOG_INFO(local_logger, "Pangu kafka brokerlist : %s", brokerlist); - TFE_LOG_INFO(local_logger, "Pangu kafka topic : %s", topic_name); - - instance->kafka_logger = tfe_kafka_logger_create(instance->en_sendlog, nic_name, brokerlist, topic_name, local_logger); - if (instance->kafka_logger == NULL) - { - TFE_LOG_ERROR(local_logger,"Pangu log init failed, error to create kafka logger."); - goto error_out; - } + instance->device_id = (const char *)tfe_bussiness_resouce_get(DEVICE_ID); + instance->kafka_logger = (tfe_kafka_logger_t *)tfe_bussiness_resouce_get(KAFKA_LOGGER); + if (instance->kafka_logger && !instance->kafka_logger->enable) + { + TFE_LOG_ERROR(local_logger, "Pangu sendlog ENABLE, but tfe kafka logger DISABLED."); + goto error_out; + } log_file_upload_para=cache_evbase_parameter_new(profile, section, local_logger); instance->log_file_upload_instance=cache_evbase_instance_new(log_file_upload_para, local_logger); diff --git a/plugin/business/traffic-mirror/src/entry.cpp b/plugin/business/traffic-mirror/src/entry.cpp index ff68ea7..60c0d49 100644 --- a/plugin/business/traffic-mirror/src/entry.cpp +++ b/plugin/business/traffic-mirror/src/entry.cpp @@ -497,7 +497,7 @@ int traffic_mirror_init(struct tfe_proxy * proxy) /* MAAT Feather, the configuration is same with pangu-http */ instance->maat_feather = maat_feather_create_with_override( - "traffic-mirror", "./conf/pangu/pangu_pxy.conf", + "traffic-mirror", "./conf/tfe/tfe.conf", "maat", "traffic_mirror", instance->nr_threads, instance->logger); if (unlikely(!instance->maat_feather)) diff --git a/resource/pangu/table_info.conf b/resource/pangu/table_info.conf index b2a2b27..2ad9c16 100644 --- a/resource/pangu/table_info.conf +++ b/resource/pangu/table_info.conf @@ -16,6 +16,7 @@ #For expr/expr_plus Table #id name type src_charset dst_charset do_merge cross_cache quick_mode 0 PXY_CTRL_COMPILE compile escape -- +<<<<<<< HEAD 1 GROUP_COMPILE_RELATION group2compile -- 2 GROUP_GROUP_RELATION group2group -- 3 TSG_OBJ_IP_ADDR ip_plus --- @@ -60,3 +61,48 @@ 40 TSG_SECURITY_DESTINATION_LOCATION virtual TSG_OBJ_GEO_LOCATION -- 41 TSG_FIELD_DOH_QNAME virtual TSG_OBJ_FQDN -- 42 TSG_FIELD_DOH_HOST virtual TSG_OBJ_FQDN -- +======= +1 GROUP_COMPILE_RELATION group -- +2 TSG_OBJ_IP_ADDR ip_plus --- +3 TSG_OBJ_URL expr UTF8 GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 0 quickoff +4 TSG_FIELD_HTTP_URL virtual TSG_OBJ_URL -- +5 TSG_OBJ_FQDN expr UTF8 GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 0 quickoff +5 TSG_OBJ_FQDN_CAT expr UTF8 GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 0 quickoff +6 TSG_FIELD_HTTP_HOST virtual TSG_OBJ_FQDN -- +7 TSG_OBJ_HTTP_SIGNATURE expr_plus UTF8 UTF8 yes 0 quickoff +8 TSG_FIELD_HTTP_REQ_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- +9 TSG_FIELD_HTTP_RES_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- +10 TSG_OBJ_KEYWORDS expr UTF8 GBK/UNICODE/UTF8 yes 128 quickoff +11 TSG_FIELD_HTTP_REQ_CONTENT virtual TSG_OBJ_KEYWORDS -- +12 TSG_FIELD_HTTP_RES_CONTENT virtual TSG_OBJ_KEYWORDS -- +13 TSG_OBJ_SUBSCRIBER_ID expr UTF8 UTF8 yes 0 quickon +14 TSG_OBJ_APP_ID expr UTF8 UTF8 yes 0 +15 TSG_SECURITY_SOURCE_ADDR virtual TSG_OBJ_IP_ADDR -- +16 TSG_SECURITY_DESTINATION_ADDR virtual TSG_OBJ_IP_ADDR -- +17 TSG_SECURITY_ADDR composition {"source":"TSG_SECURITY_SOURCE_ADDR","destination":"TSG_SECURITY_DESTINATION_ADDR"} +18 PXY_CACHE_COMPILE compile escape -- +18 PXY_CACHE_GROUP group -- +19 PXY_CACHE_HTTP_URL expr UTF8 UTF8 yes 0 quickoff +20 PXY_CACHE_HTTP_COOKIE expr UTF8 UTF8 yes 0 quickoff +21 PXY_PROFILE_TRUSTED_CA_CERT plugin {"key":1,"valid":4,"foreign":"3"} +21 PXY_EXCH_INTERMEDIA_CERT plugin {"key":1,"valid":4,"foreign":"3"} +22 PXY_OBJ_TRUSTED_CA_CRL plugin {"valid":4,"foreign":"3"} +23 TSG_PROFILE_RESPONSE_PAGES plugin {"key":1,"foreign":"4","valid":5} +24 PXY_PROFILE_HIJACK_FILES plugin {"key":1,"foreign":"5","valid":6} +25 PXY_PROFILE_INSERT_SCRIPTS plugin {"key":1,"foreign":"4","valid":6} +26 TSG_SECURITY_COMPILE plugin {"key":1,"valid":8} +27 PXY_PROFILE_TRAFFIC_MIRROR plugin {"key":1,"valid":4} +28 TSG_PROFILE_DECRYPTION plugin {"key":1,"valid":4} +29 TSG_IP_ASN_BUILT_IN ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7,"estimate_size":4194304} +30 TSG_IP_ASN_USER_DEFINED ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7,"estimate_size":4194304} +31 TSG_IP_LOCATION_BUILT_IN ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18,"estimate_size":4194304} +32 TSG_IP_LOCATION_USER_DEFINED ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18,"estimate_size":4194304} +33 TSG_OBJ_AS_NUMBER expr UTF8 UTF8/GBK yes 0 +34 TSG_SECURITY_SOURCE_ASN virtual TSG_OBJ_AS_NUMBER -- +35 TSG_SECURITY_DESTINATION_ASN virtual TSG_OBJ_AS_NUMBER -- +36 TSG_OBJ_GEO_LOCATION expr UTF8 UTF8/GBK yes 0 +37 TSG_SECURITY_SOURCE_LOCATION virtual TSG_OBJ_GEO_LOCATION -- +38 TSG_SECURITY_DESTINATION_LOCATION virtual TSG_OBJ_GEO_LOCATION -- +39 TSG_FIELD_DOH_QNAME virtual TSG_OBJ_FQDN -- +40 TSG_FIELD_DOH_HOST virtual TSG_OBJ_FQDN -- +>>>>>>> TSG-1531 tfe 代码整理, 将多插件公用的基础代码移动到 tfe init 阶段