gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-18177 Proxy删除关于FQDN Category的配置加载和扫描, Proxy适配maat_scan_group函数变更

Browse Source
This commit is contained in:
fengweihao
2024-01-03 17:26:20 +08:00
parent cbd98507a2
commit 058a5e6e66
9 changed files with 8 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/include/tfe_scan.h
View File

@@ -9,8 +9,6 @@ int tfe_scan_ip_asn(const struct tfe_stream *stream, long long *result, struct m
int hit_cnt, void *logger, char **asn_server, char **asn_client);
int tfe_scan_ip_location(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
int hit_cnt, void *logger, char **location_server, char **location_client);
int tfe_scan_fqdn_cat(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
int hit_cnt, void *logger, int table_id);
int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt, int app_id, int table_id);
int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, struct ipaddr sapp_addr);
int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, struct ipaddr sapp_addr);

74
common/src/tfe_scan.cpp
View File

@@ -79,67 +79,6 @@ int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, st
return hit_cnt_ip;
}
int tfe_scan_fqdn_cat(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
int hit_cnt, void *logger, int table_id)
{
int scan_ret = 0;
unsigned int i = 0;
uint16_t opt_out_size;
int hit_cnt_fqdn = 0;
size_t n_hit_result = 0;
unsigned int category_id_num = 0;
unsigned int category_id_val[TFE_SYMBOL_MAX] = {0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL)
{
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_NUM, (unsigned char *)&category_id_num, sizeof(category_id_num), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch fqdn cat id num from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_VAL, (unsigned char *)category_id_val, sizeof(category_id_val), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch fqdn cat id val from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
}
TFE_LOG_DEBUG(logger, "fetch fqdn cat id:%d val:%d,%d,%d,%d,%d,%d,%d,%d addr: %s", category_id_num,
(category_id_num >= 1 ? category_id_val[0] : 0),
(category_id_num >= 2 ? category_id_val[1] : 0),
(category_id_num >= 3 ? category_id_val[2] : 0),
(category_id_num >= 4 ? category_id_val[3] : 0),
(category_id_num >= 5 ? category_id_val[4] : 0),
(category_id_num >= 6 ? category_id_val[5] : 0),
(category_id_num >= 7 ? category_id_val[6] : 0),
(category_id_num >= 8 ? category_id_val[7] : 0),
stream->str_stream_info);
for (i = 0; i < category_id_num && i < 8; i++)
{
scan_ret = maat_scan_integer((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, category_id_val[i], result + hit_cnt + hit_cnt_fqdn,
MAX_SCAN_RESULT - hit_cnt - hit_cnt_fqdn, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
TFE_LOG_INFO(logger, "Scan FQDN_CAT, Hit catid: %d scan ret: %d policy_id: %lld addr: %s",
category_id_val[i], scan_ret, result[hit_cnt + hit_cnt_fqdn], stream->str_stream_info);
hit_cnt_fqdn += n_hit_result;
}
else
{
TFE_LOG_INFO(logger, "Scan FQDN_CAT, NO hit catid: %d scan ret: %d addr: %s",
category_id_val[i], scan_ret, stream->str_stream_info);
}
}
scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, result + hit_cnt + hit_cnt_fqdn,
MAX_SCAN_RESULT - hit_cnt - hit_cnt_fqdn, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_fqdn += n_hit_result;
}
return hit_cnt_fqdn;
}
int tfe_scan_ip_location(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
int hit_cnt, void *logger, char **location_server, char **location_client)
{
@@ -320,12 +259,15 @@ int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt,
int scan_ret = 0;
int hit_app_id = 0;
size_t n_hit_result = 0;
struct maat_hit_group hit_group;
struct app_id_dict *app_dict = (struct app_id_dict*)maat_plugin_table_get_ex_data((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_OBJ_APP_ID_DICT),
(const char *)&app_id, sizeof(long long));
if(app_dict!=NULL)
{
scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, &app_dict->group_id, 1, result+hit_cnt+hit_app_id,
memset(&hit_group, 0, sizeof(hit_group));
hit_group.group_id=app_dict->group_id;
scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, &hit_group, 1, result+hit_cnt+hit_app_id,
MAX_SCAN_RESULT-hit_cnt-hit_app_id, &n_hit_result, scan_mid);
if(scan_ret==MAAT_SCAN_HIT)
{
@@ -365,7 +307,7 @@ int tfe_scan_internal_exteral_addr(const struct tfe_stream *stream, long long *r
int hit_cnt_ip = 0;
size_t array_size=256;
size_t n_hit_result = 0;
long long group_id_array[256]={0};
struct maat_hit_group last_hit_groups[256] = {0};
int dir_is_e2i = get_route_dir(stream);
int scan_internal_table_id=maat_get_table_id((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), "ATTR_INTERNAL_ADDR");
@@ -374,11 +316,11 @@ int tfe_scan_internal_exteral_addr(const struct tfe_stream *stream, long long *r
if(c2s == 1) table_id = (dir_is_e2i == 1) ? scan_internal_table_id : scan_external_table_id;
if(c2s == 0) table_id = (dir_is_e2i == 0) ? scan_internal_table_id : scan_external_table_id;
size_t n_last_hit_group = maat_state_get_last_hit_group_id_cnt(scan_mid);
size_t n_last_hit_group = maat_state_get_last_hit_group_cnt(scan_mid);
if(n_last_hit_group > 0)
{
maat_state_get_last_hit_group_ids(scan_mid, group_id_array, array_size);
scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, group_id_array, array_size, result+hit_cnt+hit_cnt_ip,
maat_state_get_last_hit_groups(scan_mid, last_hit_groups, array_size);
scan_ret = maat_scan_group((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, last_hit_groups, array_size, result+hit_cnt+hit_cnt_ip,
MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
if(scan_ret == MAAT_SCAN_HIT)
{

1
conf/doh/doh.conf
View File

@@ -7,7 +7,6 @@ table_src_addr=ATTR_SOURCE_ADDR
table_dst_addr=ATTR_DESTINATION_ADDR
table_qname=ATTR_DOH_QNAME
table_host=ATTR_SERVER_FQDN
table_host_cat=ATTR_SERVER_FQDN_CAT
[kafka]
ENTRANCE_ID=0

77
conf/pangu/pangu_pxy.conf
View File

@@ -1,77 +0,0 @@
[debug]
enable_plugin=1
[log]
entrance_id=0
# default 1, if enable "en_sendlog", the iterm "tfe.conf [kafka] enable" must set 1
en_hoslog=1
en_sendlog=1
#Addresses of hos, Bucket name in hos. Format is defined by WiredLB.
cache_ip_list=10.4.35.42-46;
cache_listen_port=9098
cache_bucket_name=hos/proxy_hos_bucket
cache_token=c21f969b5f03d33d43e04f8f136e7682
#Refer to the pangu_cahche definition
max_used_memroy_size_mb=5120
cache_default_ttl_second=3600
cache_object_key_hash_switch=1
#Refer to the pangu_cahche definition
cache_store_object_way=0
redis_cache_object_size=1024000
redis_cluster_addrs=10.4.20.211:9001,10.4.20.212:9001,10.4.20.213:9001,10.4.20.214:9001,10.4.20.215:9001,10.4.20.216:9001,10.4.20.217:9001,10.4.20.218:9001
#Configs of WiredLB for Minios load balancer.
wiredlb_override=1
wiredlb_health_port=42310
wiredlb_topic=MinioFileLog
wiredlb_datacenter=k18consul-tse
wiredlb_health_port=52102
wiredlb_group=FileLog
log_fsstat_appname=tango_log_file
log_fsstat_filepath=./log/tango_log_file.fs2
log_fsstat_interval=10
log_fsstat_trig=1
log_fsstat_dst_ip=10.4.20.202
log_fsstat_dst_port=8125
[ratelimit]
#hijack flow control
enable=0
token_name=ratelimit
redis_server=192.168.40.137
redis_port=6379
redis_db_index=5
[tango_cache]
enable_cache=0
min_cache_obj_size=512
#hos ip, as wiredlb required
cache_ip_list=10.4.35.1-14;
cache_listen_port=9000
cache_bucket_name=proxybucket
max_cnnt_pipeline_num=20
#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value.
max_used_memory_size_mb=10240
#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute).
cache_default_ttl_second=3600
#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it.
cache_object_key_hash_switch=1
#Store way: 0-HOS; 1-META in REDIS, object in hos; 2-META and small object in Redis, large object in hos;
cache_store_object_way=2
#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis.
redis_cache_object_size=102400
#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
redis_cluster_addrs=10.4.35.15:9001,10.4.35.16:9001,10.4.35.17:9001,10.4.35.18:9001,10.4.35.19:9001,10.4.35.20:9001,10.4.35.21:9001,10.4.35.22:9001,10.4.35.23:9001,10.4.35.24:9001,10.4.35.25:9001,10.4.35.26:9001,10.4.35.27:9001,10.4.35.28:9001,10.4.35.29:9001,10.4.35.30:9001,10.4.35.31:9001,10.4.35.32:9001
#Configs of WiredLB for Minios load balancer.Refer to the definition at log
cache_undefined_obj=1
query_undefined_obj=0
statsd_server=10.4.20.201
statsd_port=8125
histogram_bins=0.20,0.40,0.6,0.8

7
plugin/business/doh/src/doh.cpp
View File

@@ -313,12 +313,6 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
{
hit_cnt += n_hit_result;
}
scan_ret = tfe_scan_fqdn_cat(stream, result, ctx->scan_mid, hit_cnt, g_doh_conf->local_logger, g_doh_conf->tables[TYPE_HOST_CAT].id);
if( scan_ret > 0)
{
hit_cnt += scan_ret;
}
}
// scan addr
@@ -387,7 +381,6 @@ static int doh_maat_init(const char *profile, const char *section)
MESA_load_profile_string_def(profile, section, "table_dst_addr", g_doh_conf->tables[TYPE_DST_ADDR].name, TFE_STRING_MAX, "ATTR_DESTINATION_ADDR");
MESA_load_profile_string_def(profile, section, "table_qname", g_doh_conf->tables[TYPE_QNAME].name, TFE_STRING_MAX, "ATTR_DOH_QNAME");
MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN");
MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN_CAT");
MESA_load_profile_string_def(profile, section, "table_internal_addr", g_doh_conf->tables[TYPE_INTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_INTERNAL_ADDR");
MESA_load_profile_string_def(profile, section, "table_external_addr", g_doh_conf->tables[TYPE_EXTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_EXTERNAL_ADDR");

1
plugin/business/doh/src/pub.h
View File

@@ -35,7 +35,6 @@ enum table_type
TYPE_APPID,
TYPE_QNAME,
TYPE_HOST,
TYPE_HOST_CAT,
TYPE_INTERNAL_ADDR,
TYPE_EXTERNAL_ADDR,
TYPE_MAX

7
plugin/business/tsg-http/src/tsg_http.cpp
View File

@@ -66,7 +66,6 @@ enum scan_table
PXY_CTRL_DESTINATION_ADDR,
PXY_CTRL_HTTP_URL,
PXY_CTRL_HTTP_FQDN,
PXY_CTRL_HTTP_FQDN_CAT,
PXY_CTRL_HTTP_REQ_HDR,
PXY_CTRL_HTTP_REQ_BODY,
PXY_CTRL_HTTP_RES_HDR,
@@ -1015,7 +1014,6 @@ int proxy_policy_init(const char* profile_path, const char* static_section, cons
table_name[PXY_CTRL_DESTINATION_ADDR]="ATTR_DESTINATION_ADDR";
table_name[PXY_CTRL_HTTP_URL] = "ATTR_HTTP_URL";
table_name[PXY_CTRL_HTTP_FQDN] = "ATTR_SERVER_FQDN";
table_name[PXY_CTRL_HTTP_FQDN_CAT] = "ATTR_SERVER_FQDN_CAT";
table_name[PXY_CTRL_HTTP_REQ_HDR] = "ATTR_HTTP_REQ_HDR";
table_name[PXY_CTRL_HTTP_REQ_BODY] = "ATTR_HTTP_REQ_BODY";
table_name[PXY_CTRL_HTTP_RES_HDR] = "ATTR_HTTP_RES_HDR";
@@ -2596,11 +2594,6 @@ enum proxy_action http_scan(const struct tfe_http_session * session, enum tfe_ht
{
hit_cnt += n_hit_result;
}
scan_ret = tfe_scan_fqdn_cat(stream, result, ctx->scan_mid, hit_cnt, g_proxy_rt->local_logger, g_proxy_rt->scan_table_id[PXY_CTRL_HTTP_FQDN_CAT]);
if (scan_ret > 0)
{
hit_cnt += scan_ret;
}
}
const char * str_url = session->req->req_spec.url;

27
plugin/business/tsg-http/src/tsg_logger.cpp
View File

@@ -109,7 +109,6 @@ size_t file_bucket_upload_once(struct proxy_logger* handle, char *uuid, struct e
struct proxy_logger* proxy_log_handle_create(const char* profile, const char* section, void* local_logger)
{
struct tango_cache_parameter *log_file_upload_para=NULL;
struct proxy_logger* instance=ALLOC(struct proxy_logger,1);
instance->local_logger=local_logger;
@@ -138,26 +137,6 @@ error_out:
return NULL;
}
static unsigned int proxy_log_get_fqdn_cat(struct tfe_cmsg *cmsg, unsigned int *category_id_val, size_t sz_out_value_buf)
{
int ret=0;
unsigned category_id_num=0;
uint16_t opt_out_size;
ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_NUM, (unsigned char *)&category_id_num, sizeof(category_id_num), &opt_out_size);
if (ret != 0 || category_id_num == 0)
{
return -1;
}
ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_FQDN_CAT_ID_VAL, (unsigned char *)category_id_val, sz_out_value_buf, &opt_out_size);
if (ret != 0)
{
return -1;
}
return category_id_num > 8 ? 8 : category_id_num;
}
int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
{
const struct tfe_http_session* http=log_msg->http;
@@ -200,7 +179,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time));
cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time));
unsigned int category_id_val[64]={0};
char source_subscribe_id[64]={0};
char opt_val[24]={0}; uint16_t opt_out_size;
struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream);
@@ -216,11 +194,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
{
cJSON_AddStringToObject(common_obj, "subscriber_id", source_subscribe_id);
}
ret = proxy_log_get_fqdn_cat(cmsg, category_id_val, sizeof(category_id_val));
if (ret>0)
{
cJSON_AddItemToObject(common_obj, "fqdn_category_list", cJSON_CreateIntArray((const int*)category_id_val, ret));
}
}
if (http->req)

6
resource/pangu/table_info.conf
View File

@@ -132,12 +132,6 @@
"table_name": "ATTR_SERVER_FQDN",
"table_type": "virtual",
"physical_table": "TSG_OBJ_FQDN"
},
{
"table_id": 11,
"table_name": "ATTR_SERVER_FQDN_CAT",
"table_type": "virtual",
"physical_table": "TSG_OBJ_FQDN_CAT"
},
{
"table_id":12,