diff --git a/common/include/tfe_scan.h b/common/include/tfe_scan.h index 77d6071..de93003 100644 --- a/common/include/tfe_scan.h +++ b/common/include/tfe_scan.h @@ -9,8 +9,6 @@ int tfe_scan_ip_asn(const struct tfe_stream *stream, long long *result, struct m int hit_cnt, void *logger, char **asn_server, char **asn_client); int tfe_scan_ip_location(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger, char **location_server, char **location_client); -int tfe_scan_fqdn_cat(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, - int hit_cnt, void *logger, int table_id); int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt, int app_id, int table_id); int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, struct ipaddr sapp_addr); int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, struct ipaddr sapp_addr); \ No newline at end of file diff --git a/common/src/tfe_scan.cpp b/common/src/tfe_scan.cpp index 1948526..02f31a9 100644 --- a/common/src/tfe_scan.cpp +++ b/common/src/tfe_scan.cpp @@ -79,67 +79,6 @@ int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, st return hit_cnt_ip; } -int tfe_scan_fqdn_cat(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, - int hit_cnt, void *logger, int table_id) -{ - int scan_ret = 0; - unsigned int i = 0; - uint16_t opt_out_size; - int hit_cnt_fqdn = 0; - size_t n_hit_result = 0; - unsigned int category_id_num = 0; - unsigned int category_id_val[TFE_SYMBOL_MAX] = {0}; - - struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream); - if (cmsg != NULL) - { - scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_NUM, (unsigned char *)&category_id_num, sizeof(category_id_num), &opt_out_size); - if (scan_ret != 0) - { - TFE_LOG_ERROR(logger, "fetch fqdn cat id num from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info); - } - scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_VAL, (unsigned char *)category_id_val, sizeof(category_id_val), &opt_out_size); - if (scan_ret != 0) - { - TFE_LOG_ERROR(logger, "fetch fqdn cat id val from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info); - } - } - TFE_LOG_DEBUG(logger, "fetch fqdn cat id:%d val:%d,%d,%d,%d,%d,%d,%d,%d addr: %s", category_id_num, - (category_id_num >= 1 ? category_id_val[0] : 0), - (category_id_num >= 2 ? category_id_val[1] : 0), - (category_id_num >= 3 ? category_id_val[2] : 0), - (category_id_num >= 4 ? category_id_val[3] : 0), - (category_id_num >= 5 ? category_id_val[4] : 0), - (category_id_num >= 6 ? category_id_val[5] : 0), - (category_id_num >= 7 ? category_id_val[6] : 0), - (category_id_num >= 8 ? category_id_val[7] : 0), - stream->str_stream_info); - - for (i = 0; i < category_id_num && i < 8; i++) - { - scan_ret = maat_scan_integer((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, category_id_val[i], result + hit_cnt + hit_cnt_fqdn, - MAX_SCAN_RESULT - hit_cnt - hit_cnt_fqdn, &n_hit_result, scan_mid); - if (scan_ret == MAAT_SCAN_HIT) - { - TFE_LOG_INFO(logger, "Scan FQDN_CAT, Hit catid: %d scan ret: %d policy_id: %lld addr: %s", - category_id_val[i], scan_ret, result[hit_cnt + hit_cnt_fqdn], stream->str_stream_info); - hit_cnt_fqdn += n_hit_result; - } - else - { - TFE_LOG_INFO(logger, "Scan FQDN_CAT, NO hit catid: %d scan ret: %d addr: %s", - category_id_val[i], scan_ret, stream->str_stream_info); - } - } - scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, result + hit_cnt + hit_cnt_fqdn, - MAX_SCAN_RESULT - hit_cnt - hit_cnt_fqdn, &n_hit_result, scan_mid); - if (scan_ret == MAAT_SCAN_HIT) - { - hit_cnt_fqdn += n_hit_result; - } - return hit_cnt_fqdn; -} - int tfe_scan_ip_location(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger, char **location_server, char **location_client) { @@ -320,12 +259,15 @@ int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt, int scan_ret = 0; int hit_app_id = 0; size_t n_hit_result = 0; + struct maat_hit_group hit_group; struct app_id_dict *app_dict = (struct app_id_dict*)maat_plugin_table_get_ex_data((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_OBJ_APP_ID_DICT), (const char *)&app_id, sizeof(long long)); if(app_dict!=NULL) { - scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, &app_dict->group_id, 1, result+hit_cnt+hit_app_id, + memset(&hit_group, 0, sizeof(hit_group)); + hit_group.group_id=app_dict->group_id; + scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, &hit_group, 1, result+hit_cnt+hit_app_id, MAX_SCAN_RESULT-hit_cnt-hit_app_id, &n_hit_result, scan_mid); if(scan_ret==MAAT_SCAN_HIT) { @@ -365,7 +307,7 @@ int tfe_scan_internal_exteral_addr(const struct tfe_stream *stream, long long *r int hit_cnt_ip = 0; size_t array_size=256; size_t n_hit_result = 0; - long long group_id_array[256]={0}; + struct maat_hit_group last_hit_groups[256] = {0}; int dir_is_e2i = get_route_dir(stream); int scan_internal_table_id=maat_get_table_id((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), "ATTR_INTERNAL_ADDR"); @@ -374,11 +316,11 @@ int tfe_scan_internal_exteral_addr(const struct tfe_stream *stream, long long *r if(c2s == 1) table_id = (dir_is_e2i == 1) ? scan_internal_table_id : scan_external_table_id; if(c2s == 0) table_id = (dir_is_e2i == 0) ? scan_internal_table_id : scan_external_table_id; - size_t n_last_hit_group = maat_state_get_last_hit_group_id_cnt(scan_mid); + size_t n_last_hit_group = maat_state_get_last_hit_group_cnt(scan_mid); if(n_last_hit_group > 0) { - maat_state_get_last_hit_group_ids(scan_mid, group_id_array, array_size); - scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, group_id_array, array_size, result+hit_cnt+hit_cnt_ip, + maat_state_get_last_hit_groups(scan_mid, last_hit_groups, array_size); + scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, last_hit_groups, array_size, result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if(scan_ret == MAAT_SCAN_HIT) { diff --git a/conf/doh/doh.conf b/conf/doh/doh.conf index 2bf4ea0..edecec2 100644 --- a/conf/doh/doh.conf +++ b/conf/doh/doh.conf @@ -7,7 +7,6 @@ table_src_addr=ATTR_SOURCE_ADDR table_dst_addr=ATTR_DESTINATION_ADDR table_qname=ATTR_DOH_QNAME table_host=ATTR_SERVER_FQDN -table_host_cat=ATTR_SERVER_FQDN_CAT [kafka] ENTRANCE_ID=0 diff --git a/conf/pangu/pangu_pxy.conf b/conf/pangu/pangu_pxy.conf deleted file mode 100644 index 414ddf6..0000000 --- a/conf/pangu/pangu_pxy.conf +++ /dev/null @@ -1,77 +0,0 @@ -[debug] -enable_plugin=1 - -[log] -entrance_id=0 -# default 1, if enable "en_sendlog", the iterm "tfe.conf [kafka] enable" must set 1 -en_hoslog=1 -en_sendlog=1 - -#Addresses of hos, Bucket name in hos. Format is defined by WiredLB. -cache_ip_list=10.4.35.42-46; -cache_listen_port=9098 -cache_bucket_name=hos/proxy_hos_bucket -cache_token=c21f969b5f03d33d43e04f8f136e7682 - -#Refer to the pangu_cahche definition -max_used_memroy_size_mb=5120 -cache_default_ttl_second=3600 -cache_object_key_hash_switch=1 - -#Refer to the pangu_cahche definition -cache_store_object_way=0 -redis_cache_object_size=1024000 -redis_cluster_addrs=10.4.20.211:9001,10.4.20.212:9001,10.4.20.213:9001,10.4.20.214:9001,10.4.20.215:9001,10.4.20.216:9001,10.4.20.217:9001,10.4.20.218:9001 - -#Configs of WiredLB for Minios load balancer. -wiredlb_override=1 -wiredlb_health_port=42310 -wiredlb_topic=MinioFileLog -wiredlb_datacenter=k18consul-tse -wiredlb_health_port=52102 -wiredlb_group=FileLog - -log_fsstat_appname=tango_log_file -log_fsstat_filepath=./log/tango_log_file.fs2 -log_fsstat_interval=10 -log_fsstat_trig=1 -log_fsstat_dst_ip=10.4.20.202 -log_fsstat_dst_port=8125 - -[ratelimit] -#hijack flow control -enable=0 -token_name=ratelimit -redis_server=192.168.40.137 -redis_port=6379 -redis_db_index=5 - -[tango_cache] -enable_cache=0 -min_cache_obj_size=512 -#hos ip, as wiredlb required -cache_ip_list=10.4.35.1-14; -cache_listen_port=9000 -cache_bucket_name=proxybucket - -max_cnnt_pipeline_num=20 -#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value. -max_used_memory_size_mb=10240 -#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute). -cache_default_ttl_second=3600 -#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it. -cache_object_key_hash_switch=1 -#Store way: 0-HOS; 1-META in REDIS, object in hos; 2-META and small object in Redis, large object in hos; -cache_store_object_way=2 -#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis. -redis_cache_object_size=102400 -#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object. -redis_cluster_addrs=10.4.35.15:9001,10.4.35.16:9001,10.4.35.17:9001,10.4.35.18:9001,10.4.35.19:9001,10.4.35.20:9001,10.4.35.21:9001,10.4.35.22:9001,10.4.35.23:9001,10.4.35.24:9001,10.4.35.25:9001,10.4.35.26:9001,10.4.35.27:9001,10.4.35.28:9001,10.4.35.29:9001,10.4.35.30:9001,10.4.35.31:9001,10.4.35.32:9001 - -#Configs of WiredLB for Minios load balancer.Refer to the definition at log - -cache_undefined_obj=1 -query_undefined_obj=0 -statsd_server=10.4.20.201 -statsd_port=8125 -histogram_bins=0.20,0.40,0.6,0.8 diff --git a/plugin/business/doh/src/doh.cpp b/plugin/business/doh/src/doh.cpp index f8f854a..d14f8ab 100644 --- a/plugin/business/doh/src/doh.cpp +++ b/plugin/business/doh/src/doh.cpp @@ -313,12 +313,6 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http { hit_cnt += n_hit_result; } - - scan_ret = tfe_scan_fqdn_cat(stream, result, ctx->scan_mid, hit_cnt, g_doh_conf->local_logger, g_doh_conf->tables[TYPE_HOST_CAT].id); - if( scan_ret > 0) - { - hit_cnt += scan_ret; - } } // scan addr @@ -387,7 +381,6 @@ static int doh_maat_init(const char *profile, const char *section) MESA_load_profile_string_def(profile, section, "table_dst_addr", g_doh_conf->tables[TYPE_DST_ADDR].name, TFE_STRING_MAX, "ATTR_DESTINATION_ADDR"); MESA_load_profile_string_def(profile, section, "table_qname", g_doh_conf->tables[TYPE_QNAME].name, TFE_STRING_MAX, "ATTR_DOH_QNAME"); MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN"); - MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN_CAT"); MESA_load_profile_string_def(profile, section, "table_internal_addr", g_doh_conf->tables[TYPE_INTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_INTERNAL_ADDR"); MESA_load_profile_string_def(profile, section, "table_external_addr", g_doh_conf->tables[TYPE_EXTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_EXTERNAL_ADDR"); diff --git a/plugin/business/doh/src/pub.h b/plugin/business/doh/src/pub.h index 6f830a2..2ca5271 100644 --- a/plugin/business/doh/src/pub.h +++ b/plugin/business/doh/src/pub.h @@ -35,7 +35,6 @@ enum table_type TYPE_APPID, TYPE_QNAME, TYPE_HOST, - TYPE_HOST_CAT, TYPE_INTERNAL_ADDR, TYPE_EXTERNAL_ADDR, TYPE_MAX diff --git a/plugin/business/tsg-http/src/tsg_http.cpp b/plugin/business/tsg-http/src/tsg_http.cpp index 8e34112..271bae2 100644 --- a/plugin/business/tsg-http/src/tsg_http.cpp +++ b/plugin/business/tsg-http/src/tsg_http.cpp @@ -66,7 +66,6 @@ enum scan_table PXY_CTRL_DESTINATION_ADDR, PXY_CTRL_HTTP_URL, PXY_CTRL_HTTP_FQDN, - PXY_CTRL_HTTP_FQDN_CAT, PXY_CTRL_HTTP_REQ_HDR, PXY_CTRL_HTTP_REQ_BODY, PXY_CTRL_HTTP_RES_HDR, @@ -1015,7 +1014,6 @@ int proxy_policy_init(const char* profile_path, const char* static_section, cons table_name[PXY_CTRL_DESTINATION_ADDR]="ATTR_DESTINATION_ADDR"; table_name[PXY_CTRL_HTTP_URL] = "ATTR_HTTP_URL"; table_name[PXY_CTRL_HTTP_FQDN] = "ATTR_SERVER_FQDN"; - table_name[PXY_CTRL_HTTP_FQDN_CAT] = "ATTR_SERVER_FQDN_CAT"; table_name[PXY_CTRL_HTTP_REQ_HDR] = "ATTR_HTTP_REQ_HDR"; table_name[PXY_CTRL_HTTP_REQ_BODY] = "ATTR_HTTP_REQ_BODY"; table_name[PXY_CTRL_HTTP_RES_HDR] = "ATTR_HTTP_RES_HDR"; @@ -2596,11 +2594,6 @@ enum proxy_action http_scan(const struct tfe_http_session * session, enum tfe_ht { hit_cnt += n_hit_result; } - scan_ret = tfe_scan_fqdn_cat(stream, result, ctx->scan_mid, hit_cnt, g_proxy_rt->local_logger, g_proxy_rt->scan_table_id[PXY_CTRL_HTTP_FQDN_CAT]); - if (scan_ret > 0) - { - hit_cnt += scan_ret; - } } const char * str_url = session->req->req_spec.url; diff --git a/plugin/business/tsg-http/src/tsg_logger.cpp b/plugin/business/tsg-http/src/tsg_logger.cpp index 2321306..8e35de2 100644 --- a/plugin/business/tsg-http/src/tsg_logger.cpp +++ b/plugin/business/tsg-http/src/tsg_logger.cpp @@ -109,7 +109,6 @@ size_t file_bucket_upload_once(struct proxy_logger* handle, char *uuid, struct e struct proxy_logger* proxy_log_handle_create(const char* profile, const char* section, void* local_logger) { - struct tango_cache_parameter *log_file_upload_para=NULL; struct proxy_logger* instance=ALLOC(struct proxy_logger,1); instance->local_logger=local_logger; @@ -138,26 +137,6 @@ error_out: return NULL; } -static unsigned int proxy_log_get_fqdn_cat(struct tfe_cmsg *cmsg, unsigned int *category_id_val, size_t sz_out_value_buf) -{ - int ret=0; - unsigned category_id_num=0; - uint16_t opt_out_size; - - ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_NUM, (unsigned char *)&category_id_num, sizeof(category_id_num), &opt_out_size); - if (ret != 0 || category_id_num == 0) - { - return -1; - } - ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_VAL, (unsigned char *)category_id_val, sz_out_value_buf, &opt_out_size); - if (ret != 0) - { - return -1; - } - - return category_id_num > 8 ? 8 : category_id_num; -} - int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) { const struct tfe_http_session* http=log_msg->http; @@ -200,7 +179,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time)); cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time)); - unsigned int category_id_val[64]={0}; char source_subscribe_id[64]={0}; char opt_val[24]={0}; uint16_t opt_out_size; struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream); @@ -216,11 +194,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) { cJSON_AddStringToObject(common_obj, "subscriber_id", source_subscribe_id); } - ret = proxy_log_get_fqdn_cat(cmsg, category_id_val, sizeof(category_id_val)); - if (ret>0) - { - cJSON_AddItemToObject(common_obj, "fqdn_category_list", cJSON_CreateIntArray((const int*)category_id_val, ret)); - } } if (http->req) diff --git a/resource/pangu/table_info.conf b/resource/pangu/table_info.conf index 75a7258..5c2aca9 100644 --- a/resource/pangu/table_info.conf +++ b/resource/pangu/table_info.conf @@ -132,12 +132,6 @@ "table_name": "ATTR_SERVER_FQDN", "table_type": "virtual", "physical_table": "TSG_OBJ_FQDN" - }, - { - "table_id": 11, - "table_name": "ATTR_SERVER_FQDN_CAT", - "table_type": "virtual", - "physical_table": "TSG_OBJ_FQDN_CAT" }, { "table_id":12,