gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG Master 共享部分扫描结果给 TFE

Browse Source 
*	TSG-2847 - TFE 新增 cmsg 共享 TSG Master 的扫描结果
*	TSG-2851 - Pangu 从 cmsg 获取 TSG Master 的扫描结果
*	TSG-2852 - DOH   从 cmsg 获取 TSG Master 的扫描结果
*	TSG-2849 - TFE DOH 功能支持 SUBID
*	合并 pangu/doh 的 maat scan 的代码为 tfe_scan
This commit is contained in:
luwenpeng
2020-08-10 18:13:44 +08:00
parent ddb1ccba12
commit fbc7b5a715
16 changed files with 351 additions and 718 deletions
Download Patch File Download Diff File Expand all files Collapse all files

231
plugin/business/pangu-http/src/pangu_http.cpp
View File

@@ -8,6 +8,7 @@
#include <tfe_http.h>
#include <tfe_plugin.h>
#include <tfe_resource.h>
#include <tfe_scan.h>
#include <MESA/Maat_rule.h>
#include <MESA/MESA_handle_logger.h>
@@ -28,7 +29,6 @@
#include "ratelimiter.h"
#define MAX_SCAN_RESULT 16
#define MAX_EDIT_ZONE_NUM 64
enum pangu_action //Bigger action number is prior.
@@ -64,7 +64,6 @@ enum scan_table
PXY_CTRL_HTTP_REQ_BODY,
PXY_CTRL_HTTP_RES_HDR,
PXY_CTRL_HTTP_RES_BODY,
PXY_CTRL_SUBSCRIBE_ID,
PXY_CTRL_APP_ID,
__SCAN_TABLE_MAX
};
@@ -127,8 +126,6 @@ struct policy_action_param
struct pangu_rt
{
Maat_feather_t maat;
Maat_feather_t dyn_maat;
int subscriber_id_table_id;
struct pangu_logger * send_logger;
void * local_logger;
int log_level;
@@ -297,57 +294,6 @@ void trusted_CA_update_finish_cb(void* u_para)
}
}
static int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len)
{
const char* seps=" \t";
char* saveptr=NULL, *subtoken=NULL, *str=NULL;
char* dup_line=tfe_strdup(line);
int i=0, ret=-1;
for (str = dup_line; ; str = NULL)
{
subtoken = strtok_r(str, seps, &saveptr);
if (subtoken == NULL)
break;
if(i==column_seq-1)
{
*offset=subtoken-dup_line;
*len=strlen(subtoken);
ret=0;
break;
}
i++;
}
free(dup_line);
return ret;
}
void subscribe_id_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA* to, MAAT_PLUGIN_EX_DATA* from, long argl, void* argp)
{
*to = tfe_strdup((char*)*from);
return;
}
void subscribe_id_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
int ret=0;
size_t subscribe_id_offset, len;
ret=get_column_pos(table_line, 4, &subscribe_id_offset, &len);
if(ret<0)
{
TFE_LOG_ERROR(g_pangu_rt->local_logger, "Add subscribe ID faild: %s", table_line);
return;
}
*ad=ALLOC(char, len+1);
memcpy(*ad, table_line+subscribe_id_offset, len);
TFE_LOG_INFO(g_pangu_rt->local_logger, "Add subscribe ID: %s", (char*)*ad);
return;
}
void subscribe_id_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
TFE_LOG_INFO(g_pangu_rt->local_logger, "Delete subscribe ID: %s", (char*)*ad);
free(*ad);
*ad=NULL;
}
static enum manipulate_action manipulate_action_str2idx(const char *action_str)
{
const char *clue_action_map[__MA_ACTION_MAX];
@@ -792,7 +738,6 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons
table_name[PXY_CTRL_HTTP_REQ_BODY] = "TSG_FIELD_HTTP_REQ_CONTENT";
table_name[PXY_CTRL_HTTP_RES_HDR] = "TSG_FIELD_HTTP_RES_HDR";
table_name[PXY_CTRL_HTTP_RES_BODY] = "TSG_FIELD_HTTP_RES_CONTENT";
table_name[PXY_CTRL_SUBSCRIBE_ID] = "TSG_OBJ_SUBSCRIBER_ID";
table_name[PXY_CTRL_APP_ID] = "TSG_OBJ_APP_ID";
for (int i = 0; i < __SCAN_TABLE_MAX; i++)
{
@@ -841,22 +786,6 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons
}
}
g_pangu_rt->dyn_maat = (Maat_feather_t)tfe_bussiness_resouce_get(DYNAMINC_MAAT);
g_pangu_rt->subscriber_id_table_id=Maat_table_register(g_pangu_rt->dyn_maat, "TSG_DYN_SUBSCRIBER_IP");
ret=Maat_plugin_EX_register(g_pangu_rt->dyn_maat,
g_pangu_rt->subscriber_id_table_id,
subscribe_id_new_cb,
subscribe_id_free_cb,
subscribe_id_dup_cb,
NULL,
0,
NULL);
if(ret!=0)
{
TFE_LOG_ERROR(NULL, "Pangu HTTP Dynamic Maat TSG_DYN_SUBSCRIBER_IP EX data register failed.");
goto error_out;
}
error_out:
return ret;
}
@@ -2405,151 +2334,6 @@ void cache_write(const struct tfe_http_session * session, enum tfe_http_event ev
}
}
int http_ip_subscribe_id_scan(struct Maat_rule_t *result, const char* sip, const char* dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx)
{
int scan_ret=0, hit_cnt_ip=0;
char* source_subscribe_id=NULL, *dest_subscribe_id=NULL;
source_subscribe_id=(char*)Maat_plugin_get_EX_data(g_pangu_rt->dyn_maat, g_pangu_rt->subscriber_id_table_id, sip);
dest_subscribe_id=(char*)Maat_plugin_get_EX_data(g_pangu_rt->dyn_maat, g_pangu_rt->subscriber_id_table_id, dip);
if(source_subscribe_id!=NULL)
{
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_SUBSCRIBE_ID],
CHARSET_UTF8, source_subscribe_id, strlen(source_subscribe_id),
result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
}
if(dest_subscribe_id!=NULL)
{
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_SUBSCRIBE_ID],
CHARSET_UTF8, dest_subscribe_id, strlen(dest_subscribe_id),
result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
}
if(source_subscribe_id!=NULL)
free(source_subscribe_id);
if(dest_subscribe_id!=NULL)
free(dest_subscribe_id);
return hit_cnt_ip;
}
int http_ip_location_scan(struct Maat_rule_t *result, struct ip_address *sip, struct ip_address *dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx )
{
int scan_ret=0, hit_cnt_ip=0;
char buff[TFE_STRING_MAX]={0};
struct ip_data_table* ip_location_client=NULL, *ip_location_server=NULL;
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_USER_DEFINED), sip, (void **)&ip_location_client, 1);
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_USER_DEFINED), dip, (void **)&ip_location_server, 1);
if (ip_location_client == NULL)
{
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_BUILT_IN), sip, (void **)&ip_location_client, 1);
}
if (ip_location_server == NULL)
{
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_BUILT_IN), dip, (void **)&ip_location_server, 1);
}
if(ip_location_server!=NULL)
{
snprintf(buff, sizeof(buff), "%s.%s.", ip_location_server->country_full, ip_location_server->city_full);
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
memset(buff,0,sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", ip_location_server->city_full, ip_location_server->province_full, ip_location_server->country_full);
ctx->ip_ctx.location_server=tfe_strdup(buff);
}
if(ip_location_client!=NULL)
{
snprintf(buff, sizeof(buff), "%s.%s.", ip_location_client->country_full, ip_location_client->city_full);
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result+hit_cnt+hit_cnt_ip, NULL, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
memset(buff,0,sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", ip_location_client->city_full, ip_location_client->province_full, ip_location_client->country_full);
ctx->ip_ctx.location_client=tfe_strdup(buff);
}
if(ip_location_server)
ip_table_free(ip_location_server);
if(ip_location_client)
ip_table_free(ip_location_client);
return hit_cnt_ip;
}
int http_ip_asn_scan(struct Maat_rule_t *result, struct ip_address* sip, struct ip_address* dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx)
{
int scan_ret=0, hit_cnt_ip=0;
char buff[TFE_STRING_MAX]={0};
struct ip_data_table* ip_asn_client=NULL, *ip_asn_server=NULL;
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_USER_DEFINED), sip, (void **)&ip_asn_client, 1);
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_USER_DEFINED), dip, (void **)&ip_asn_server, 1);
if (ip_asn_client == NULL)
{
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_BUILT_IN), sip, (void **)&ip_asn_client, 1);
}
if (ip_asn_server == NULL)
{
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_BUILT_IN), dip, (void **)&ip_asn_server, 1);
}
if(ip_asn_server!=NULL)
{
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_ASN),
CHARSET_UTF8, ip_asn_server->asn, strlen(ip_asn_server->asn),
result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
snprintf(buff, sizeof(buff), "%s(%s)", ip_asn_server->asn, ip_asn_server->organization);
ctx->ip_ctx.asn_server=tfe_strdup(buff);
}
if(ip_asn_client!=NULL)
{
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_ASN),
CHARSET_UTF8, ip_asn_client->asn, strlen(ip_asn_client->asn),
result+hit_cnt+hit_cnt_ip, NULL, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
snprintf(buff, sizeof(buff), "%s(%s)", ip_asn_client->asn, ip_asn_client->organization);
ctx->ip_ctx.asn_client=tfe_strdup(buff);
}
if(ip_asn_server)
ip_table_free(ip_asn_server);
if(ip_asn_client)
ip_table_free(ip_asn_client);
return hit_cnt_ip;
}
void pangu_on_http_begin(const struct tfe_stream * stream,
const struct tfe_http_session * session, unsigned int thread_id, void ** pme)
{
@@ -2562,30 +2346,23 @@ void pangu_on_http_begin(const struct tfe_stream * stream,
struct Maat_rule_t result[MAX_SCAN_RESULT];
struct ipaddr sapp_addr;
int hit_cnt = 0, scan_ret=0;
UNUSED int tmp=0;
assert(ctx == NULL);
ATOMIC_INC(&(g_pangu_rt->stat_val[STAT_SESSION]));
ctx = pangu_http_ctx_new(thread_id);
char* addr_string=tfe_stream_addr_to_str(stream->addr);
const char* sip=NULL, *dip=NULL;
tmp=tfe_stream_addr_str_split(addr_string, &sip, NULL, &dip, NULL);
assert(tmp==0);
scan_ret = http_ip_subscribe_id_scan(result, sip, dip, hit_cnt, thread_id, ctx);
scan_ret = tfe_scan_subscribe_id(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_pangu_rt->local_logger, addr_string);
if(scan_ret>0)
{
hit_cnt+=scan_ret;
}
struct ip_address dest_ip, source_ip;
tfe_stream_addr_to_address(stream->addr, &dest_ip, &source_ip);
scan_ret = http_ip_location_scan(result, &source_ip, &dest_ip, hit_cnt, thread_id, ctx);
scan_ret = tfe_scan_ip_location(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_pangu_rt->local_logger, addr_string, &(ctx->ip_ctx.location_server), &(ctx->ip_ctx.location_client));
if(scan_ret>0)
{
hit_cnt+=scan_ret;
}
scan_ret = http_ip_asn_scan(result, &source_ip, &dest_ip, hit_cnt, thread_id, ctx);
scan_ret = tfe_scan_ip_asn(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_pangu_rt->local_logger, addr_string, &(ctx->ip_ctx.asn_server), &(ctx->ip_ctx.asn_client));
if(scan_ret>0)
{
hit_cnt+=scan_ret;