gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-17777 优化App ID的扫描处理逻辑, HTTP多次扫描支持非配置

Browse Source 
TSG-17797 TFE适配虚拟表名变更
This commit is contained in:
fengweihao
2023-11-24 15:26:23 +08:00
parent d3e49bf461
commit 72f7290ccd
6 changed files with 252 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
common/src/tfe_resource.cpp
View File

@@ -303,11 +303,11 @@ static char* create_effective_device_tag(const char *profile, const char *sectio
}
static struct maat_table_info maat_pub_tables[TABLE_TYPE_MAX] = {
{0, "TSG_SECURITY_SOURCE_ASN"},
{0, "TSG_SECURITY_DESTINATION_ASN"},
{0, "TSG_SECURITY_SOURCE_LOCATION"},
{0, "TSG_SECURITY_DESTINATION_LOCATION"},
{0, "TSG_OBJ_SUBSCRIBER_ID"}};
{0, "ATTR_SOURCE_ASN"},
{0, "ATTR_DESTINATION_ASN"},
{0, "ATTR_SOURCE_LOCATION"},
{0, "ATTR_DESTINATION_LOCATION"},
{0, "ATTR_SUBSCRIBER_ID"}};
static int register_maat_table()
{

11
common/src/tfe_scan.cpp
View File

@@ -118,6 +118,13 @@ int tfe_scan_fqdn_cat(const struct tfe_stream *stream, long long *result, struct
category_id_val[i], scan_ret, stream->str_stream_info);
}
}
scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), table_id, result + hit_cnt + hit_cnt_fqdn,
MAX_SCAN_RESULT - hit_cnt - hit_cnt_fqdn, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_fqdn += n_hit_result;
}
return hit_cnt_fqdn;
}
@@ -254,13 +261,13 @@ int tfe_scan_ip_asn(const struct tfe_stream *stream, long long *result, struct m
&n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_SOURCE_ASN, Hit asn: %s scan ret: %d policy_id: %lld addr: %s",
TFE_LOG_INFO(logger, "Scan ATTR_SOURCE_ASN, Hit asn: %s scan ret: %d policy_id: %lld addr: %s",
src_asn, scan_ret, result[hit_cnt + hit_cnt_ip], stream->str_stream_info);
hit_cnt_ip += n_hit_result;
}
else
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_SOURCE_ASN, NO hit asn: %s scan ret: %d addr: %s",
TFE_LOG_INFO(logger, "Scan ATTR_SOURCE_ASN, NO hit asn: %s scan ret: %d addr: %s",
src_asn, scan_ret, stream->str_stream_info);
}

10
conf/doh/doh.conf
View File

@@ -2,10 +2,12 @@
enable=1
[maat]
table_appid=TSG_OBJ_APP_ID
table_addr=TSG_SECURITY_ADDR
table_qname=TSG_FIELD_DOH_QNAME
table_host=TSG_FIELD_DOH_HOST
table_appid=ATTR_APP_ID
table_src_addr=ATTR_SOURCE_ADDR
table_dst_addr=ATTR_DESTINATION_ADDR
table_qname=ATTR_DOH_QNAME
table_host=ATTR_DOH_HOST
table_host_cat=ATTR_DOH_HOST_CAT
[kafka]
ENTRANCE_ID=0

92
plugin/business/doh/src/doh.cpp
View File

@@ -10,6 +10,32 @@ extern void increase_redirect_policy_hit_num(void);
#define REQ_METHOD_IS_GET(method) ((method == TFE_HTTP_METHOD_GET) ? 1 : 0)
#define REQ_METHOD_IS_POST(method) ((method == TFE_HTTP_METHOD_POST) ? 1 : 0)
struct doh_action_param
{
int ref_cnt;
int action;
char *message;
char *position;
float enforcement_ratio;
int profile_id;
int status_code;
size_t n_rule;
void *repl_rule;
size_t e_rule;
void *elem_rule;
struct doh_maat_rule_t hit_rule;
pthread_mutex_t lock;
};
struct doh_app_id_dict
{
int ref_cnt;
int app_id;
long long int group_id;
pthread_mutex_t lock;
};
struct dns_str2idx
{
int index;
@@ -101,23 +127,6 @@ static cJSON *doh_get_answer_records(struct doh_ctx *ctx, cJSON *object, int qty
return NULL;
}
struct doh_action_param
{
int ref_cnt;
int action;
char *message;
char *position;
float enforcement_ratio;
int profile_id;
int status_code;
size_t n_rule;
void *repl_rule;
size_t e_rule;
void *elem_rule;
struct doh_maat_rule_t hit_rule;
pthread_mutex_t lock;
};
void doh_action_param_free_cb(int table_id, void **ad, long argl, void *argp)
{
if(*ad==NULL)
@@ -142,6 +151,29 @@ void doh_action_param_free_cb(int table_id, void **ad, long argl, void *argp)
return;
}
void doh_app_dict_table_free_cb(int table_id, void **ad, long argl, void* argp)
{
if(*ad==NULL)
{
return;
}
struct doh_app_id_dict *app_dict=(struct doh_app_id_dict *)(*ad);
pthread_mutex_lock(&(app_dict->lock));
app_dict->ref_cnt--;
if(app_dict->ref_cnt>0)
{
pthread_mutex_unlock(&(app_dict->lock));
return;
}
pthread_mutex_unlock(&(app_dict->lock));
pthread_mutex_destroy(&(app_dict->lock));
FREE(&app_dict);
*ad=NULL;
return;
}
static void doh_get_cheat_data(long long p_result, int qtype, struct doh_ctx *ctx, const char *str_stream_info)
{
int i;
@@ -353,8 +385,16 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
}
}
// scan appid
scan_ret=maat_scan_integer(g_doh_conf->maat, g_doh_conf->tables[TYPE_APPID].id, app_id, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &n_hit_result, ctx->scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
int table_id=maat_get_table_id(g_doh_conf->maat, "APP_ID_DICT");
if(table_id < 0)
{
return;
}
struct doh_app_id_dict *app_dict = (struct doh_app_id_dict *)maat_plugin_table_get_ex_data(g_doh_conf->maat, table_id, (const char *)&app_id, sizeof(long long));
if(app_dict!=NULL)
{
scan_ret = maat_scan_group(g_doh_conf->maat, g_doh_conf->tables[TYPE_APPID].id, &app_dict->group_id, 1, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &n_hit_result, ctx->scan_mid);
if(scan_ret==MAAT_SCAN_HIT)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, Hit proto: %d scan ret: %d policy_id: %lld addr: %s",
g_doh_conf->tables[TYPE_APPID].name, app_id, scan_ret, result[hit_cnt], stream->str_stream_info);
@@ -365,6 +405,8 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, NO hit proto: %d scan ret: %d addr: %s",
g_doh_conf->tables[TYPE_APPID].name, app_id, scan_ret, stream->str_stream_info);
}
doh_app_dict_table_free_cb(0, (void **)&app_dict, 0, NULL);
}
// scan qname
scan_ret = maat_scan_string(g_doh_conf->maat, g_doh_conf->tables[TYPE_QNAME].id, qname, strlen(qname),
@@ -396,12 +438,12 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
static int doh_maat_init(const char *profile, const char *section)
{
g_doh_conf->maat = (struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT);
MESA_load_profile_string_def(profile, section, "table_appid", g_doh_conf->tables[TYPE_APPID].name, TFE_STRING_MAX, "TSG_OBJ_APP_ID");
MESA_load_profile_string_def(profile, section, "table_src_addr", g_doh_conf->tables[TYPE_SRC_ADDR].name, TFE_STRING_MAX, "TSG_SECURITY_SOURCE_ADDR");
MESA_load_profile_string_def(profile, section, "table_dst_addr", g_doh_conf->tables[TYPE_DST_ADDR].name, TFE_STRING_MAX, "TSG_SECURITY_DESTINATION_ADDR");
MESA_load_profile_string_def(profile, section, "table_qname", g_doh_conf->tables[TYPE_QNAME].name, TFE_STRING_MAX, "TSG_FIELD_DOH_QNAME");
MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "TSG_FIELD_DOH_HOST");
MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "TSG_FIELD_DOH_HOST_CAT");
MESA_load_profile_string_def(profile, section, "table_appid", g_doh_conf->tables[TYPE_APPID].name, TFE_STRING_MAX, "ATTR_APP_ID");
MESA_load_profile_string_def(profile, section, "table_src_addr", g_doh_conf->tables[TYPE_SRC_ADDR].name, TFE_STRING_MAX, "ATTR_SOURCE_ADDR");
MESA_load_profile_string_def(profile, section, "table_dst_addr", g_doh_conf->tables[TYPE_DST_ADDR].name, TFE_STRING_MAX, "ATTR_DESTINATION_ADDR");
MESA_load_profile_string_def(profile, section, "table_qname", g_doh_conf->tables[TYPE_QNAME].name, TFE_STRING_MAX, "ATTR_DOH_QNAME");
MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_DOH_HOST");
MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_DOH_HOST_CAT");
for (int i = 0; i < TYPE_MAX; i++)
{

129
plugin/business/tsg-http/src/tsg_http.cpp
View File

@@ -103,9 +103,19 @@ enum manipulate_profile_table
POLICY_PROFILE_TABLE_INSERT,
POLICY_PROFILE_TABLE_HIJACK,
POLICY_PROFILE_TABLE_LUA,
POLICY_PROFILE_TABLE_APP_ID,
POLICY_PROFILE_TABLE_MAX
};
struct app_id_dict
{
int ref_cnt;
int app_id;
long long int group_id;
pthread_mutex_t lock;
};
struct manipulate_profile
{
int profile_id;
@@ -958,6 +968,76 @@ void ma_profile_table_dup_cb(int table_id, void **to, void **from, long argl, vo
*to=ply_obj;
}
void app_dict_table_new_cb(const char *table_name, int table_id, const char* key, const char* table_line, void **ad, long argl, void* argp)
{
int ret=0;
size_t offset=0, len=0;
char *app_id_str=NULL, *group_id_str=NULL;
struct app_id_dict *app_dict=ALLOC(struct app_id_dict, 1);
ret = maat_helper_read_column(table_line, 1, &offset, &len);
if(ret >= 0)
{
app_id_str=ALLOC(char, len+1);
memcpy(app_id_str, table_line+offset, len);
app_dict->app_id=atoi(app_id_str);
FREE(&app_id_str);
}
ret = maat_helper_read_column(table_line, 18, &offset, &len);
if(ret >= 0)
{
group_id_str=ALLOC(char, len+1);
memcpy(group_id_str, table_line+offset, len);
app_dict->group_id=atoll(group_id_str);
FREE(&group_id_str);
}
app_dict->ref_cnt=1;
pthread_mutex_init(&(app_dict->lock), NULL);
*ad=app_dict;
return;
}
void app_dict_table_free_cb(int table_id, void **ad, long argl, void* argp)
{
if(*ad==NULL)
{
return;
}
struct app_id_dict *app_dict=(struct app_id_dict *)(*ad);
pthread_mutex_lock(&(app_dict->lock));
app_dict->ref_cnt--;
if(app_dict->ref_cnt>0)
{
pthread_mutex_unlock(&(app_dict->lock));
return;
}
pthread_mutex_unlock(&(app_dict->lock));
pthread_mutex_destroy(&(app_dict->lock));
FREE(&app_dict);
*ad=NULL;
return;
}
void app_id_dict_free(struct app_id_dict *app_dict)
{
app_dict_table_free_cb(0, (void **)&app_dict, 0, NULL);
}
void app_dict_table_dup_cb(int table_id, void **to, void **from, long argl, void* argp)
{
struct app_id_dict *app_dict=(struct app_id_dict *)(*from);
pthread_mutex_lock(&(app_dict->lock));
app_dict->ref_cnt++;
pthread_mutex_unlock(&(app_dict->lock));
*to=app_dict;
return;
}
int maat_table_init(const char* table_name,
maat_start_callback_t *start, maat_update_callback_t *update, maat_finish_callback_t *finish,
void *u_para)
@@ -1017,16 +1097,16 @@ int proxy_policy_init(const char* profile_path, const char* static_section, cons
g_proxy_rt->feather = (struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT);
const char * table_name[__SCAN_TABLE_MAX];
table_name[PXY_CTRL_SOURCE_ADDR] = "TSG_SECURITY_SOURCE_ADDR";
table_name[PXY_CTRL_DESTINATION_ADDR]="TSG_SECURITY_DESTINATION_ADDR";
table_name[PXY_CTRL_HTTP_URL] = "TSG_FIELD_HTTP_URL";
table_name[PXY_CTRL_HTTP_FQDN] = "TSG_FIELD_HTTP_HOST";
table_name[PXY_CTRL_HTTP_FQDN_CAT] = "TSG_FIELD_HTTP_HOST_CAT";
table_name[PXY_CTRL_HTTP_REQ_HDR] = "TSG_FIELD_HTTP_REQ_HDR";
table_name[PXY_CTRL_HTTP_REQ_BODY] = "TSG_FIELD_HTTP_REQ_BODY";
table_name[PXY_CTRL_HTTP_RES_HDR] = "TSG_FIELD_HTTP_RES_HDR";
table_name[PXY_CTRL_HTTP_RES_BODY] = "TSG_FIELD_HTTP_RES_BODY";
table_name[PXY_CTRL_APP_ID] = "TSG_OBJ_APP_ID";
table_name[PXY_CTRL_SOURCE_ADDR] = "ATTR_SOURCE_ADDR";
table_name[PXY_CTRL_DESTINATION_ADDR]="ATTR_DESTINATION_ADDR";
table_name[PXY_CTRL_HTTP_URL] = "ATTR_HTTP_URL";
table_name[PXY_CTRL_HTTP_FQDN] = "ATTR_HTTP_HOST";
table_name[PXY_CTRL_HTTP_FQDN_CAT] = "ATTR_HTTP_HOST_CAT";
table_name[PXY_CTRL_HTTP_REQ_HDR] = "ATTR_HTTP_REQ_HDR";
table_name[PXY_CTRL_HTTP_REQ_BODY] = "ATTR_HTTP_REQ_BODY";
table_name[PXY_CTRL_HTTP_RES_HDR] = "ATTR_HTTP_RES_HDR";
table_name[PXY_CTRL_HTTP_RES_BODY] = "ATTR_HTTP_RES_BODY";
table_name[PXY_CTRL_APP_ID] = "ATTR_APP_ID";
for (int i = 0; i < __SCAN_TABLE_MAX; i++)
{
g_proxy_rt->scan_table_id[i] = maat_get_table_id(g_proxy_rt->feather, table_name[i]);
@@ -1044,6 +1124,13 @@ int proxy_policy_init(const char* profile_path, const char* static_section, cons
policy_action_param_dup,
0, NULL);
g_proxy_rt->plolicy_table_id[POLICY_PROFILE_TABLE_APP_ID]=maat_get_table_id(g_proxy_rt->feather, "APP_ID_DICT");
maat_plugin_table_ex_schema_register(g_proxy_rt->feather, "APP_ID_DICT",
app_dict_table_new_cb,
app_dict_table_free_cb,
app_dict_table_dup_cb,
0, NULL);
ret = maat_table_init("PXY_PROFILE_TRUSTED_CA_CERT",
trusted_CA_update_start_cb,
trusted_CA_update_cert_cb,
@@ -2739,6 +2826,12 @@ enum proxy_action http_scan(const struct tfe_http_session * session, enum tfe_ht
hit_cnt += n_hit_result;
}
}
scan_ret = maat_scan_not_logic(g_proxy_rt->feather, table_id, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt,
&n_hit_result, ctx->scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt += n_hit_result;
}
}
if ((events & EV_HTTP_REQ_BODY_BEGIN) | (events & EV_HTTP_RESP_BODY_BEGIN))
@@ -2761,6 +2854,12 @@ enum proxy_action http_scan(const struct tfe_http_session * session, enum tfe_ht
if ((events & EV_HTTP_REQ_BODY_END) | (events & EV_HTTP_RESP_BODY_END))
{
scan_ret = maat_scan_not_logic(g_proxy_rt->feather, table_id, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt,
&n_hit_result, ctx->scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt += n_hit_result;
}
maat_stream_free(ctx->sp);
ctx->sp = NULL;
}
@@ -3124,12 +3223,18 @@ void proxy_on_http_begin(const struct tfe_stream * stream,
{
hit_cnt+=scan_ret;
}
int scan_val=67;
scan_ret=maat_scan_integer(g_proxy_rt->feather, g_proxy_rt->scan_table_id[PXY_CTRL_APP_ID], scan_val, result+hit_cnt, MAX_SCAN_RESULT-hit_cnt, &n_hit_result, ctx->scan_mid);
long long app_id=67;
struct app_id_dict *app_dict = (struct app_id_dict*)maat_plugin_table_get_ex_data(g_proxy_rt->feather, g_proxy_rt->plolicy_table_id[POLICY_PROFILE_TABLE_APP_ID], (const char *)&app_id, sizeof(long long));
if(app_dict!=NULL)
{
scan_ret = maat_scan_group(g_proxy_rt->feather, g_proxy_rt->scan_table_id[PXY_CTRL_APP_ID], &app_dict->group_id, 1, result+hit_cnt, MAX_SCAN_RESULT-hit_cnt, &n_hit_result, ctx->scan_mid);
if(scan_ret==MAAT_SCAN_HIT)
{
hit_cnt+=n_hit_result;
}
app_id_dict_free(app_dict);
}
addr_tfe2sapp(stream->addr, &sapp_addr);
if (sapp_addr.addrtype == ADDR_TYPE_IPV4)

55
resource/pangu/table_info.conf
View File

@@ -97,7 +97,7 @@
},
{
"table_id":7,
"table_name":"TSG_FIELD_HTTP_URL",
"table_name":"ATTR_HTTP_URL",
"table_type":"virtual",
"physical_table": "TSG_OBJ_URL"
},
@@ -129,13 +129,13 @@
},
{
"table_id":10,
"table_name":"TSG_FIELD_HTTP_HOST",
"table_name":"ATTR_HTTP_HOST",
"table_type":"virtual",
"physical_table": "TSG_OBJ_FQDN"
},
{
"table_id":11,
"table_name":"TSG_FIELD_HTTP_HOST_CAT",
"table_name":"ATTR_HTTP_HOST_CAT",
"table_type":"virtual",
"physical_table": "TSG_OBJ_FQDN_CAT"
},
@@ -156,13 +156,13 @@
},
{
"table_id":13,
"table_name":"TSG_FIELD_HTTP_REQ_HDR",
"table_name":"ATTR_HTTP_REQ_HDR",
"table_type":"virtual",
"physical_table": "TSG_OBJ_HTTP_SIGNATURE"
},
{
"table_id":14,
"table_name":"TSG_FIELD_HTTP_RES_HDR",
"table_name":"ATTR_HTTP_RES_HDR",
"table_type":"virtual",
"physical_table": "TSG_OBJ_HTTP_SIGNATURE"
},
@@ -182,13 +182,13 @@
},
{
"table_id":16,
"table_name":"TSG_FIELD_HTTP_REQ_BODY",
"table_name":"ATTR_HTTP_REQ_BODY",
"table_type":"virtual",
"physical_table": "TSG_OBJ_KEYWORDS"
},
{
"table_id":17,
"table_name":"TSG_FIELD_HTTP_RES_BODY",
"table_name":"ATTR_HTTP_RES_BODY",
"table_type":"virtual",
"physical_table": "TSG_OBJ_KEYWORDS"
},
@@ -220,13 +220,13 @@
},
{
"table_id":20,
"table_name":"TSG_SECURITY_SOURCE_ADDR",
"table_name":"ATTR_SOURCE_ADDR",
"table_type":"virtual",
"physical_table": "TSG_OBJ_IP"
},
{
"table_id":21,
"table_name":"TSG_SECURITY_DESTINATION_ADDR",
"table_name":"ATTR_DESTINATION_ADDR",
"table_type":"virtual",
"physical_table": "TSG_OBJ_IP"
},
@@ -411,13 +411,13 @@
},
{
"table_id":37,
"table_name":"TSG_SECURITY_SOURCE_ASN",
"table_name":"ATTR_SOURCE_ASN",
"table_type":"virtual",
"physical_table": "TSG_OBJ_AS_NUMBER"
},
{
"table_id":38,
"table_name":"TSG_SECURITY_DESTINATION_ASN",
"table_name":"ATTR_DESTINATION_ASN",
"table_type":"virtual",
"physical_table": "TSG_OBJ_AS_NUMBER"
},
@@ -437,31 +437,31 @@
},
{
"table_id":40,
"table_name":"TSG_SECURITY_SOURCE_LOCATION",
"table_name":"ATTR_SOURCE_LOCATION",
"table_type":"virtual",
"physical_table": "TSG_OBJ_GEO_LOCATION"
},
{
"table_id":41,
"table_name":"TSG_SECURITY_DESTINATION_LOCATION",
"table_name":"ATTR_DESTINATION_LOCATION",
"table_type":"virtual",
"physical_table": "TSG_OBJ_GEO_LOCATION"
},
{
"table_id":42,
"table_name":"TSG_FIELD_DOH_QNAME",
"table_name":"ATTR_DOH_QNAME",
"table_type":"virtual",
"physical_table": "TSG_OBJ_FQDN"
},
{
"table_id":43,
"table_name":"TSG_FIELD_DOH_HOST",
"table_name":"ATTR_DOH_HOST",
"table_type":"virtual",
"physical_table": "TSG_OBJ_FQDN"
},
{
"table_id":44,
"table_name":"TSG_FIELD_DOH_HOST_CAT",
"table_name":"ATTR_DOH_HOST_CAT",
"table_type":"virtual",
"physical_table": "TSG_OBJ_FQDN_CAT"
},
@@ -505,5 +505,28 @@
"key":1,
"key_type":"pointer"
}
},
{
"table_id": 49,
"table_name": "APP_ID_DICT",
"table_type": "plugin",
"valid_column": 18,
"custom": {
"key": 1,
"key_type": "integer",
"key_len": 8
}
},
{
"table_id":50,
"table_name":"ATTR_APP_ID",
"table_type":"virtual",
"physical_table": "APP_ID_DICT"
},
{
"table_id":51,
"table_name":"ATTR_SUBSCRIBER_ID",
"table_type":"virtual",
"physical_table": "TSG_OBJ_SUBSCRIBER_ID"
}
]