diff --git a/bin/maat.conf b/bin/maat.conf index a6b9da6..ec29619 100644 --- a/bin/maat.conf +++ b/bin/maat.conf @@ -1,6 +1,5 @@ [STATIC] MAAT_MODE=1 -#EFFECTIVE_FLAG= STAT_SWITCH=1 PERF_SWITCH=1 TABLE_INFO=tsgconf/tsg_static_tableinfo.conf @@ -13,10 +12,10 @@ REDIS_INDEX=0 JSON_CFG_FILE=tsgconf/tsg_maat.json INC_CFG_DIR=tsgrule/inc/index/ FULL_CFG_DIR=tsgrule/full/index/ +EFFECTIVE_RANGE_FILE=/opt/tsg/etc/tsg_device_tag.json [DYNAMIC] MAAT_MODE=1 -#EFFECTIVE_FLAG= STAT_SWITCH=1 PERF_SWITCH=1 TABLE_INFO=tsgconf/tsg_dynamic_tableinfo.conf @@ -29,3 +28,7 @@ REDIS_INDEX=1 JSON_CFG_FILE=tsgconf/tsg_maat.json INC_CFG_DIR=tsgrule/inc/index/ FULL_CFG_DIR=tsgrule/full/index/ +EFFECTIVE_RANGE_FILE=/opt/tsg/etc/tsg_device_tag.json + +[MAAT] +ACCEPT_TAGS={"tags":[{"tag":"device_id","value":"device_1"}]} diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 10382af..7bc0755 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -42,7 +42,7 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL; #endif -char TSG_MASTER_VERSION_20200706=0; +char TSG_MASTER_VERSION_20200724=0; const char *tsg_conffile="tsgconf/main.conf"; g_tsg_para_t g_tsg_para; diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 6f0e25a..1d366e3 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -199,13 +199,14 @@ void location_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DAT MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "IP_LOCATION", - "Dup(table_id: %d) location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d", + "Dup(table_id: %d) location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s province: %s city: %s ref_cnt: %d", table_id, location->geoname_id, location->start_ip, location->end_ip, location->continent_full, location->country_full, + location->province_full, location->city_full, location->ref_cnt ); @@ -268,13 +269,14 @@ void location_new_data(int table_id, const char* key, const char* table_line, MA MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "IP_LOCATION", - "Add(table_id: %d) ip location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d", + "Add(table_id: %d) ip location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s province: %s city: %s ref_cnt: %d", table_id, location->geoname_id, location->start_ip, location->end_ip, location->continent_full, location->country_full, + location->province_full, location->city_full, location->ref_cnt ); @@ -289,13 +291,14 @@ void location_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "IP_LOCATION", - "Delete(table_id: %d) location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d", + "Delete(table_id: %d) location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s province: %s city: %s ref_cnt: %d", table_id, location->geoname_id, location->start_ip, location->end_ip, location->continent_full, location->country_full, + location->province_full, location->city_full, location->ref_cnt ); @@ -384,20 +387,21 @@ void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, vo return; } - static Maat_feather_t init_maat_feather(const char* conffile, char* instance_name, char *module, void *logger) { unsigned short redis_port = 0; int ret=0,scan_detail=0,effect_interval=60; Maat_feather_t _maat_feather=NULL; int factor=0, redis_port_num=0,redis_index=0; + char effective_range_filename[1024]={0}; char redis_ip[16]={0}, effective_flag[1024]={0}; int maat_mode=0,maat_stat_on=0,maat_perf_on=0,thread_max=0; char json_cfg_file[MAX_PATH_LEN]={0},maat_stat_file[MAX_PATH_LEN]={0}; char table_info[MAX_PATH_LEN]={0},inc_cfg_dir[MAX_PATH_LEN]={0},ful_cfg_dir[MAX_PATH_LEN]={0}; - memset(effective_flag, 0, sizeof(effective_flag)); - MESA_load_profile_string_def(conffile,module,"EFFECTIVE_FLAG",effective_flag, sizeof(effective_flag),""); + memset(effective_flag, 0, sizeof(effective_flag)); + MESA_load_profile_string_def(conffile, module, "EFFECTIVE_RANGE_FILE", effective_range_filename, sizeof(effective_range_filename),"./tsgconf/maat.conf"); + MESA_load_profile_string_def(conffile, "MAAT", "ACCEPT_TAGS", effective_flag, sizeof(effective_flag),""); MESA_load_profile_int_def(conffile, module,"MAAT_MODE", &(maat_mode),0); MESA_load_profile_int_def(conffile, module,"STAT_SWITCH", &(maat_stat_on),1); @@ -443,6 +447,10 @@ static Maat_feather_t init_maat_feather(const char* conffile, char* instance_nam ret=Maat_set_feather_opt(_maat_feather,MAAT_OPT_ACCEPT_TAGS,effective_flag, strlen(effective_flag)+1); assert(ret>=0); } + else + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "EFFECTIVE_RANGE", "Effective range is empty, please check %s", effective_range_filename); + } Maat_set_feather_opt(_maat_feather,MAAT_OPT_INSTANCE_NAME,instance_name, strlen(instance_name)+1); if(maat_mode==1) { diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 5af348a..d62854b 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -19,7 +19,7 @@ #include "tsg_send_log.h" #include "tsg_send_log_internal.h" -char TSG_SEND_LOG_VERSION_20200720=0; +char TSG_SEND_LOG_VERSION_20200724=0; struct tsg_log_instance_t *g_tsg_log_instance; @@ -206,14 +206,20 @@ int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD if(internal_label->client_location!=NULL) { location=internal_label->client_location; - snprintf(buff, sizeof(buff), "%s,%s,%s", location->city_full, location->province_full, location->country_full); + snprintf(buff, sizeof(buff), "%s,%s,%s", (memcmp(location->city_full, "null", 4) ? "" : location->city_full), + (memcmp(location->province_full, "null", 4) ? "" : location->province_full), + (memcmp(location->country_full, "null", 4) ? "" : location->country_full) + ); TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_LOCATION].name, (void *)buff, TLD_TYPE_STRING); } if(internal_label->server_location!=NULL) { location=internal_label->server_location; - snprintf(buff, sizeof(buff), "%s,%s,%s", location->city_full, location->province_full, location->country_full); + snprintf(buff, sizeof(buff), "%s,%s,%s", (memcmp(location->city_full, "null", 4) ? "" : location->city_full), + (memcmp(location->province_full, "null", 4) ? "" : location->province_full), + (memcmp(location->country_full, "null", 4) ? "" : location->country_full) + ); TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_LOCATION].name, (void *)buff, TLD_TYPE_STRING); } } @@ -236,6 +242,8 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle struct layer_addr_ipv6 *ipv6=NULL; char server_ip[MAX_IPV4_LEN*8]={0}; char client_ip[MAX_IPV4_LEN*8]={0}; + struct tcp_flow_stat *tflow_project=NULL; + struct udp_flow_stat *uflow_project=NULL; struct TLD_handle_t *_handle=handle; struct tsg_log_instance_t *_instance=instance; @@ -285,11 +293,32 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle TLD_append(_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)(long)a_stream->dir, TLD_TYPE_LONG); TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)(long)addr_type, TLD_TYPE_LONG); - - TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->clientpktnum, TLD_TYPE_LONG); - TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->clientbytes, TLD_TYPE_LONG); - TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->serverpktnum, TLD_TYPE_LONG); - TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->serverbytes, TLD_TYPE_LONG); + + switch(a_stream->type) + { + case STREAM_TYPE_TCP: + tflow_project=(struct tcp_flow_stat *)project_req_get_struct(a_stream, _instance->tcp_flow_project_id); + if(tflow_project!=NULL) + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)tflow_project->S2C_all_pkt, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)tflow_project->S2C_all_byte, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)tflow_project->C2S_all_pkt, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)tflow_project->C2S_all_byte, TLD_TYPE_LONG); + } + break; + case STREAM_TYPE_UDP: + uflow_project=(struct udp_flow_stat *)project_req_get_struct(a_stream, _instance->udp_flow_project_id); + if(uflow_project!=NULL) + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)uflow_project->S2C_pkt, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)uflow_project->S2C_byte, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)uflow_project->C2S_pkt, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)uflow_project->C2S_byte, TLD_TYPE_LONG); + } + break; + default: + break; + } if(a_stream!=NULL && a_stream->ptcpdetail!=NULL) { @@ -454,6 +483,23 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) MESA_load_profile_string_def(conffile, "TSG_LOG", "COMMON_FIELD_FILE", _instance->common_field_file, sizeof(_instance->common_field_file), NULL); MESA_load_profile_string_def(conffile, "TSG_LOG", "BROKER_LIST", _instance->broker_list, sizeof(_instance->broker_list), NULL); + MESA_load_profile_string_def(conffile, "TSG_LOG", "TCP_LABEL", _instance->tcp_label, sizeof(_instance->tcp_label), "tcp_flow_stat"); + MESA_load_profile_string_def(conffile, "TSG_LOG", "UDP_LABEL", _instance->udp_label, sizeof(_instance->udp_label), "udp_flow_stat"); + + _instance->tcp_flow_project_id=project_customer_register(_instance->tcp_label, "struct"); + _instance->udp_flow_project_id=project_customer_register(_instance->udp_label, "struct"); + if(_instance->tcp_flow_project_id<0 || _instance->udp_flow_project_id<0) + { + MESA_handle_runtime_log(_instance->logger, + RLOG_LV_FATAL, + "TCP_OR_UDP_LABEL", + "project_customer_register is error, tcp_label: %s udp_label: %s, please check etc/project.conf", + _instance->tcp_label, + _instance->udp_label + ); + } + + MESA_load_profile_string_def(conffile, "TSG_LOG", "NIC_NAME", nic_name, sizeof(nic_name), "eth0"); ret=MESA_get_dev_ipv4(nic_name, (int *)&local_ip_nr); if(ret<0) diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index c855f4e..93dff80 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -87,7 +87,11 @@ struct tsg_log_instance_t int mode; int max_service; int internal_project_id; + int tcp_flow_project_id; + int udp_flow_project_id; void *logger; + char tcp_label[MAX_STRING_LEN]; + char udp_label[MAX_STRING_LEN]; char common_field_file[MAX_STRING_LEN*4]; char broker_list[MAX_STRING_LEN*4]; char local_ip_str[MAX_IPV4_LEN];