TSG-18177 Proxy删除关于FQDN Category的配置加载和扫描, Proxy适配maat_scan_group函数变更

common/include/tfe_scan.h

@@ -9,8 +9,6 @@ int tfe_scan_ip_asn(const struct tfe_stream *stream, long long *result, struct m
                     int hit_cnt, void *logger, char **asn_server, char **asn_client);
 int tfe_scan_ip_location(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
                          int hit_cnt, void *logger, char **location_server, char **location_client);
-int tfe_scan_fqdn_cat(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
-                      int hit_cnt, void *logger, int table_id);
 int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt, int app_id, int table_id);
 int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, struct ipaddr sapp_addr);
 int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, struct ipaddr sapp_addr);

common/src/tfe_scan.cpp

@@ -79,67 +79,6 @@ int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, st
     return hit_cnt_ip;
 }
 
-int tfe_scan_fqdn_cat(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
-                      int hit_cnt, void *logger, int table_id)
-{
-	int scan_ret = 0;
-    unsigned int i = 0;
-	uint16_t opt_out_size;
-	int hit_cnt_fqdn = 0;
-	size_t n_hit_result = 0;
-	unsigned int category_id_num = 0;
-	unsigned int category_id_val[TFE_SYMBOL_MAX] = {0};
-
-    struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
-	if (cmsg != NULL)
-    {
-        scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_NUM, (unsigned char *)&category_id_num, sizeof(category_id_num), &opt_out_size);
-        if (scan_ret != 0)
-        {
-            TFE_LOG_ERROR(logger, "fetch fqdn cat id num from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
-        }
-        scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_VAL, (unsigned char *)category_id_val, sizeof(category_id_val), &opt_out_size);
-        if (scan_ret != 0)
-        {
-            TFE_LOG_ERROR(logger, "fetch fqdn cat id val from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
-        }
-    }
-    TFE_LOG_DEBUG(logger, "fetch fqdn cat id:%d val:%d,%d,%d,%d,%d,%d,%d,%d addr: %s", category_id_num,
-                  (category_id_num >= 1 ? category_id_val[0] : 0),
-                  (category_id_num >= 2 ? category_id_val[1] : 0),
-                  (category_id_num >= 3 ? category_id_val[2] : 0),
-                  (category_id_num >= 4 ? category_id_val[3] : 0),
-                  (category_id_num >= 5 ? category_id_val[4] : 0),
-                  (category_id_num >= 6 ? category_id_val[5] : 0),
-                  (category_id_num >= 7 ? category_id_val[6] : 0),
-                  (category_id_num >= 8 ? category_id_val[7] : 0),
-                  stream->str_stream_info);
-
-    for (i = 0; i < category_id_num && i < 8; i++)
-    {
-		scan_ret = maat_scan_integer((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, category_id_val[i], result + hit_cnt + hit_cnt_fqdn,
-								     MAX_SCAN_RESULT - hit_cnt - hit_cnt_fqdn, &n_hit_result, scan_mid);
-		if (scan_ret == MAAT_SCAN_HIT)
-		{
-            TFE_LOG_INFO(logger, "Scan FQDN_CAT, Hit catid: %d scan ret: %d policy_id: %lld addr: %s",
-                         category_id_val[i], scan_ret, result[hit_cnt + hit_cnt_fqdn], stream->str_stream_info);
-            hit_cnt_fqdn += n_hit_result;
-		}
-        else
-        {
-            TFE_LOG_INFO(logger, "Scan FQDN_CAT, NO hit catid: %d scan ret: %d addr: %s",
-                         category_id_val[i], scan_ret, stream->str_stream_info);
-        }
-	}
-	scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, result + hit_cnt + hit_cnt_fqdn,
-								  MAX_SCAN_RESULT - hit_cnt - hit_cnt_fqdn, &n_hit_result, scan_mid);
-	if (scan_ret == MAAT_SCAN_HIT)
-	{
-		hit_cnt_fqdn += n_hit_result;
-	}
-	return hit_cnt_fqdn;
-}
-
 int tfe_scan_ip_location(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
                          int hit_cnt, void *logger, char **location_server, char **location_client)
 {
@@ -320,12 +259,15 @@ int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt,
     int scan_ret = 0;
     int hit_app_id = 0;
     size_t n_hit_result = 0;
+    struct maat_hit_group hit_group;
  
 	struct app_id_dict *app_dict = (struct app_id_dict*)maat_plugin_table_get_ex_data((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_OBJ_APP_ID_DICT),
                                     (const char *)&app_id, sizeof(long long));
 	if(app_dict!=NULL)
 	{
-		scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, &app_dict->group_id, 1, result+hit_cnt+hit_app_id,
+        memset(&hit_group, 0, sizeof(hit_group));
+        hit_group.group_id=app_dict->group_id;
+		scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, &hit_group, 1, result+hit_cnt+hit_app_id,
                                     MAX_SCAN_RESULT-hit_cnt-hit_app_id, &n_hit_result, scan_mid);
 		if(scan_ret==MAAT_SCAN_HIT)
 		{
@@ -365,7 +307,7 @@ int tfe_scan_internal_exteral_addr(const struct tfe_stream *stream, long long *r
     int hit_cnt_ip = 0;
     size_t array_size=256;
     size_t n_hit_result = 0;
-    long long group_id_array[256]={0};
+    struct maat_hit_group last_hit_groups[256] = {0};
     
     int dir_is_e2i = get_route_dir(stream);
     int scan_internal_table_id=maat_get_table_id((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), "ATTR_INTERNAL_ADDR");
@@ -374,11 +316,11 @@ int tfe_scan_internal_exteral_addr(const struct tfe_stream *stream, long long *r
     if(c2s == 1)  table_id = (dir_is_e2i == 1) ? scan_internal_table_id : scan_external_table_id;
     if(c2s == 0)  table_id = (dir_is_e2i == 0) ? scan_internal_table_id : scan_external_table_id;
 
-    size_t n_last_hit_group = maat_state_get_last_hit_group_id_cnt(scan_mid);
+    size_t n_last_hit_group = maat_state_get_last_hit_group_cnt(scan_mid);
     if(n_last_hit_group > 0)
     {
-        maat_state_get_last_hit_group_ids(scan_mid, group_id_array, array_size);
+        maat_state_get_last_hit_groups(scan_mid, last_hit_groups, array_size);
-        scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, group_id_array, array_size, result+hit_cnt+hit_cnt_ip,
+        scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, last_hit_groups, array_size, result+hit_cnt+hit_cnt_ip,
                                     MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
         if(scan_ret == MAAT_SCAN_HIT)
         {

conf/doh/doh.conf

@@ -7,7 +7,6 @@ table_src_addr=ATTR_SOURCE_ADDR
 table_dst_addr=ATTR_DESTINATION_ADDR
 table_qname=ATTR_DOH_QNAME
 table_host=ATTR_SERVER_FQDN
-table_host_cat=ATTR_SERVER_FQDN_CAT
 
 [kafka]
 ENTRANCE_ID=0

conf/pangu/pangu_pxy.conf

@@ -1,77 +0,0 @@
-[debug]
-enable_plugin=1
-
-[log]
-entrance_id=0
-# default 1, if enable "en_sendlog", the iterm "tfe.conf [kafka] enable" must set 1
-en_hoslog=1
-en_sendlog=1
-
-#Addresses of hos, Bucket name in hos. Format is defined by WiredLB.
-cache_ip_list=10.4.35.42-46;
-cache_listen_port=9098
-cache_bucket_name=hos/proxy_hos_bucket
-cache_token=c21f969b5f03d33d43e04f8f136e7682
-
-#Refer to the pangu_cahche definition
-max_used_memroy_size_mb=5120
-cache_default_ttl_second=3600
-cache_object_key_hash_switch=1
-
-#Refer to the pangu_cahche definition
-cache_store_object_way=0
-redis_cache_object_size=1024000
-redis_cluster_addrs=10.4.20.211:9001,10.4.20.212:9001,10.4.20.213:9001,10.4.20.214:9001,10.4.20.215:9001,10.4.20.216:9001,10.4.20.217:9001,10.4.20.218:9001
-
-#Configs of WiredLB for Minios load balancer.
-wiredlb_override=1
-wiredlb_health_port=42310
-wiredlb_topic=MinioFileLog
-wiredlb_datacenter=k18consul-tse
-wiredlb_health_port=52102
-wiredlb_group=FileLog
-
-log_fsstat_appname=tango_log_file
-log_fsstat_filepath=./log/tango_log_file.fs2
-log_fsstat_interval=10
-log_fsstat_trig=1
-log_fsstat_dst_ip=10.4.20.202
-log_fsstat_dst_port=8125
-
-[ratelimit]
-#hijack flow control
-enable=0
-token_name=ratelimit
-redis_server=192.168.40.137
-redis_port=6379
-redis_db_index=5
-
-[tango_cache]
-enable_cache=0
-min_cache_obj_size=512
-#hos ip, as wiredlb required
-cache_ip_list=10.4.35.1-14;
-cache_listen_port=9000
-cache_bucket_name=proxybucket
-
-max_cnnt_pipeline_num=20
-#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value.
-max_used_memory_size_mb=10240
-#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute).
-cache_default_ttl_second=3600
-#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it.
-cache_object_key_hash_switch=1
-#Store way: 0-HOS; 1-META in REDIS, object in hos; 2-META and small object in Redis, large object in hos;
-cache_store_object_way=2
-#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis.
-redis_cache_object_size=102400
-#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
-redis_cluster_addrs=10.4.35.15:9001,10.4.35.16:9001,10.4.35.17:9001,10.4.35.18:9001,10.4.35.19:9001,10.4.35.20:9001,10.4.35.21:9001,10.4.35.22:9001,10.4.35.23:9001,10.4.35.24:9001,10.4.35.25:9001,10.4.35.26:9001,10.4.35.27:9001,10.4.35.28:9001,10.4.35.29:9001,10.4.35.30:9001,10.4.35.31:9001,10.4.35.32:9001
-
-#Configs of WiredLB for Minios load balancer.Refer to the definition at log
-
-cache_undefined_obj=1
-query_undefined_obj=0
-statsd_server=10.4.20.201
-statsd_port=8125
-histogram_bins=0.20,0.40,0.6,0.8

plugin/business/doh/src/doh.cpp

@@ -313,12 +313,6 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
         {
 			hit_cnt += n_hit_result;
         }
-
-        scan_ret = tfe_scan_fqdn_cat(stream, result, ctx->scan_mid, hit_cnt, g_doh_conf->local_logger, g_doh_conf->tables[TYPE_HOST_CAT].id);
-        if( scan_ret > 0)
-		{
-			hit_cnt += scan_ret;
-		}
 	}
 
 	// scan addr
@@ -387,7 +381,6 @@ static int doh_maat_init(const char *profile, const char *section)
 	MESA_load_profile_string_def(profile, section, "table_dst_addr", g_doh_conf->tables[TYPE_DST_ADDR].name, TFE_STRING_MAX, "ATTR_DESTINATION_ADDR");
 	MESA_load_profile_string_def(profile, section, "table_qname", g_doh_conf->tables[TYPE_QNAME].name, TFE_STRING_MAX, "ATTR_DOH_QNAME");
 	MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN");
-	MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN_CAT");
 	MESA_load_profile_string_def(profile, section, "table_internal_addr", g_doh_conf->tables[TYPE_INTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_INTERNAL_ADDR");
 	MESA_load_profile_string_def(profile, section, "table_external_addr", g_doh_conf->tables[TYPE_EXTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_EXTERNAL_ADDR");
 

plugin/business/doh/src/pub.h

@@ -35,7 +35,6 @@ enum table_type
 	TYPE_APPID,
 	TYPE_QNAME,
 	TYPE_HOST,
-    TYPE_HOST_CAT,
 	TYPE_INTERNAL_ADDR,
 	TYPE_EXTERNAL_ADDR,
 	TYPE_MAX

plugin/business/tsg-http/src/tsg_http.cpp

@@ -66,7 +66,6 @@ enum scan_table
 	PXY_CTRL_DESTINATION_ADDR,
 	PXY_CTRL_HTTP_URL,
 	PXY_CTRL_HTTP_FQDN,
-	PXY_CTRL_HTTP_FQDN_CAT,
 	PXY_CTRL_HTTP_REQ_HDR,
 	PXY_CTRL_HTTP_REQ_BODY,
 	PXY_CTRL_HTTP_RES_HDR,
@@ -1015,7 +1014,6 @@ int proxy_policy_init(const char* profile_path, const char* static_section, cons
 	table_name[PXY_CTRL_DESTINATION_ADDR]="ATTR_DESTINATION_ADDR";
 	table_name[PXY_CTRL_HTTP_URL] = "ATTR_HTTP_URL";
 	table_name[PXY_CTRL_HTTP_FQDN] = "ATTR_SERVER_FQDN";
-	table_name[PXY_CTRL_HTTP_FQDN_CAT] = "ATTR_SERVER_FQDN_CAT";
 	table_name[PXY_CTRL_HTTP_REQ_HDR] = "ATTR_HTTP_REQ_HDR";
 	table_name[PXY_CTRL_HTTP_REQ_BODY] = "ATTR_HTTP_REQ_BODY";
 	table_name[PXY_CTRL_HTTP_RES_HDR] = "ATTR_HTTP_RES_HDR";
@@ -2596,11 +2594,6 @@ enum proxy_action http_scan(const struct tfe_http_session * session, enum tfe_ht
 			{
 				hit_cnt += n_hit_result;
 			}
-            scan_ret = tfe_scan_fqdn_cat(stream, result, ctx->scan_mid, hit_cnt, g_proxy_rt->local_logger, g_proxy_rt->scan_table_id[PXY_CTRL_HTTP_FQDN_CAT]);
-            if (scan_ret > 0)
-			{
-				hit_cnt += scan_ret;
-			}
         }
 
 		const char * str_url = session->req->req_spec.url;

plugin/business/tsg-http/src/tsg_logger.cpp

@@ -109,7 +109,6 @@ size_t file_bucket_upload_once(struct proxy_logger* handle, char *uuid, struct e
 
 struct proxy_logger* proxy_log_handle_create(const char* profile, const char* section, void* local_logger)
 {
-	struct tango_cache_parameter *log_file_upload_para=NULL;
 	struct proxy_logger* instance=ALLOC(struct proxy_logger,1);
 	instance->local_logger=local_logger;
 
@@ -138,26 +137,6 @@ error_out:
 	return NULL;
 }
 
-static unsigned int proxy_log_get_fqdn_cat(struct tfe_cmsg *cmsg, unsigned int *category_id_val, size_t sz_out_value_buf)
-{
-	int ret=0;
-    unsigned category_id_num=0;
-	uint16_t opt_out_size;
-
-    ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_NUM, (unsigned char *)&category_id_num, sizeof(category_id_num), &opt_out_size);
-    if (ret != 0 || category_id_num == 0)
-    {
-        return -1;
-    }
-    ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_VAL, (unsigned char *)category_id_val, sz_out_value_buf, &opt_out_size);
-    if (ret != 0)
-    {
-        return -1;
-    }
-
-	return category_id_num > 8 ? 8 : category_id_num;
-}
-
 int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 {
 	const struct tfe_http_session* http=log_msg->http;
@@ -200,7 +179,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 	cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time));
 	cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time));
 
-	unsigned int category_id_val[64]={0};
 	char source_subscribe_id[64]={0};
 	char opt_val[24]={0}; uint16_t opt_out_size;
 	struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream);
@@ -216,11 +194,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 		{
 			cJSON_AddStringToObject(common_obj, "subscriber_id", source_subscribe_id);
 		}
-		ret = proxy_log_get_fqdn_cat(cmsg, category_id_val, sizeof(category_id_val));
-		if (ret>0)
-		{
-			cJSON_AddItemToObject(common_obj, "fqdn_category_list", cJSON_CreateIntArray((const int*)category_id_val, ret));
-		}
 	}
 
 	if (http->req)

resource/pangu/table_info.conf

@@ -132,12 +132,6 @@
         "table_name": "ATTR_SERVER_FQDN",
         "table_type": "virtual",
         "physical_table": "TSG_OBJ_FQDN"
-    },
-    {
-        "table_id": 11,
-        "table_name": "ATTR_SERVER_FQDN_CAT",
-        "table_type": "virtual",
-        "physical_table": "TSG_OBJ_FQDN_CAT"
     },
 	{
         "table_id":12,