gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-9422 TSG-9554 TSG-9470 修复元素编辑MARK问题, 代理支持4级或以上定位库

Browse Source
This commit is contained in:
fengweihao
2022-02-11 09:57:33 +08:00
parent e2be64b67b
commit 9ddc606ac5
5 changed files with 156 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/include/tfe_cmsg.h
View File

@@ -81,6 +81,8 @@ enum tfe_cmsg_tlv_type
TFE_CMSG_DST_IP_LOCATION_PROVINE, // string max size 256
TFE_CMSG_SRC_IP_LOCATION_CITY, // string max size 256
TFE_CMSG_DST_IP_LOCATION_CITY, // string max size 256
TFE_CMSG_SRC_IP_LOCATION_SUBDIVISION,
TFE_CMSG_DST_IP_LOCATION_SUBDIVISION,
/* SSL ja3 fingerprint */
TFE_CMSG_SSL_CLIENT_JA3_FINGERPRINT, // string max size 32

51
common/src/tfe_scan.cpp
View File

@@ -134,6 +134,8 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re
char dst_provine[TFE_STRING_MAX] = {0};
char src_country[TFE_STRING_MAX] = {0};
char dst_country[TFE_STRING_MAX] = {0};
char src_subdivsion[TFE_STRING_MAX] = {0};
char dst_subdivsion[TFE_STRING_MAX] = {0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL)
{
@@ -166,14 +168,32 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch dst city from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_IP_LOCATION_SUBDIVISION, (unsigned char *)src_subdivsion, sizeof(src_subdivsion), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch src subdivsion from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_IP_LOCATION_SUBDIVISION, (unsigned char *)dst_subdivsion, sizeof(dst_subdivsion), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch dst subdivsion from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
}
TFE_LOG_DEBUG(logger, "fetch src country:%s provine:%s city:%s; dst country:%s provine:%s city:%s addr: %s", src_country, src_provine, src_city, dst_country, dst_provine, dst_city, stream->str_stream_info);
TFE_LOG_DEBUG(logger, "fetch src country:%s provine:%s city:%s; subdivsion:%s, dst country:%s provine:%s city:%s subdivsion:%s addr: %s", src_country, src_provine, src_city, src_subdivsion, dst_country, dst_provine, dst_city, dst_subdivsion, stream->str_stream_info);
if (strlen(dst_country) || strlen(dst_city))
{
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s.%s.", dst_country, dst_city);
if(strlen(dst_subdivsion))
{
snprintf(buff, sizeof(buff),"%s.%s.%s.%s.", dst_country, dst_provine, dst_city, dst_subdivsion);
}
else
{
snprintf(buff, sizeof(buff), "%s.%s.", dst_country, dst_city);
}
scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
@@ -190,13 +210,27 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re
buff, scan_ret, stream->str_stream_info);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", dst_city, dst_provine, dst_country);
if(strlen(dst_subdivsion))
{
snprintf(buff, sizeof(buff), "%s,%s,%s,%s", dst_country, dst_provine, dst_city, dst_subdivsion);
}
else
{
snprintf(buff, sizeof(buff), "%s,%s,%s", dst_country, dst_provine, dst_city);
}
*location_server = tfe_strdup(buff);
}
if (strlen(src_country) || strlen(src_city))
{
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s.%s.", src_country, src_city);
if(strlen(src_subdivsion))
{
snprintf(buff, sizeof(buff), "%s.%s.%s.%s.", src_country, src_provine, src_city, src_subdivsion);
}
else
{
snprintf(buff, sizeof(buff), "%s.%s.", src_country, src_city);
}
scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
@@ -214,7 +248,14 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", src_city, src_provine, src_country);
if(strlen(src_subdivsion))
{
snprintf(buff, sizeof(buff), "%s,%s,%s,%s", src_country, src_provine, src_city, src_subdivsion);
}
else
{
snprintf(buff, sizeof(buff), "%s,%s,%s", src_country, src_provine, src_city);
}
*location_client = tfe_strdup(buff);
}

104
plugin/business/pangu-http/src/edit_element.cpp
View File

@@ -5,6 +5,11 @@
#include <stdlib.h>
#include <assert.h>
#if 0
#define PCRE2_CODE_UNIT_WIDTH 8
#include <pcre2.h>
#endif
#include <libxml/tree.h>
#include <libxml/xmlsave.h>
#include <libxml/HTMLparser.h>
@@ -13,7 +18,7 @@
#include "edit_element.h"
int cjson_element_foreach(cJSON *a, int *depth, int *step, int *step_level, char **node, const struct edit_element_rule * rules, int *match_num, int loop);
static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match);
static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match,size_t mark_tag);
size_t parse_json_output_unformatted(const char * in, size_t in_sz, const struct edit_element_rule * rules, char** out);
enum search_scope scope_name_to_id(const char * name)
@@ -52,6 +57,36 @@ int match_start_indicator(xmlNodePtr parent, char * start_indicator)
return 0;
}
#if 0
int match_string(const char * in, size_t in_sz, char *contained_keyword)
{
assert(strlen(contained_keyword) != 0);
int error=0;
PCRE2_SIZE erroffset=0;
const PCRE2_SPTR pattern = (PCRE2_SPTR)contained_keyword;
uint32_t pcre2_options = PCRE2_UTF;
pcre2_code *re = pcre2_compile(pattern, strlen(contained_keyword), pcre2_options, &error, &erroffset, 0);
if(!re)
{
return 0;
}
pcre2_match_data *match_data = pcre2_match_data_create_from_pattern(re, NULL);
int rc = 0;
const PCRE2_SPTR subject = (PCRE2_SPTR)in;
rc = pcre2_match(re, subject, in_sz, 0, 0, match_data, NULL);
pcre2_match_data_free(match_data);
pcre2_code_free(re);
return rc;
}
#endif
int construct_cjson_by_treatment(cJSON *a_element, char **node, int *step, int *step_array_level, const struct edit_element_rule * rules)
{
const char *element_treatment=rules->element_treatment;
@@ -151,6 +186,12 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP
}
}
#if 0
if(match_string((char *)node->content, strlen((char *)node->content), rules->contained_keyword) < 0)
{
return 0;
}
#endif
if(strcasestr((char *)node->content, rules->contained_keyword) == NULL)
{
return 0;
@@ -201,12 +242,8 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP
break;
}
if(*n_parent < 16)
{
parent_array[*n_parent] = parent;
*n_parent = *n_parent+1;
}
parent_array[0] = parent;
*n_parent = *n_parent+1;
*match =1;
break;
}
@@ -281,6 +318,9 @@ int cjson_dump_string(cJSON *a, int *depth, const struct edit_element_rule * rul
{
int xret=0;
#if 0
if((a->valuestring != NULL) && (match_string(a->valuestring, strlen(a->valuestring), rules->contained_keyword) > 0))
#endif
if((a->valuestring != NULL) && strcasestr(a->valuestring, rules->contained_keyword))
{
if(*depth != -1)
@@ -345,20 +385,20 @@ static void html_namespace_list(xmlNsPtr ns)
}
}
static void html_attr_list(const struct edit_element_rule * rules, xmlAttrPtr attr, xmlNodePtr *parent_array, size_t *n_parent, int *match)
static void html_attr_list(const struct edit_element_rule * rules, xmlAttrPtr attr, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
{
while (attr != NULL)
{
if (attr->children != NULL)
{
html_node_list(rules, attr->children, parent_array, n_parent, match);
html_node_list(rules, attr->children, parent_array, n_parent, match, mark_tag);
}
attr = attr->next;
}
}
static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match)
static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
{
switch (node->type)
{
@@ -385,7 +425,7 @@ static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePt
if ((node->type == XML_ELEMENT_NODE) && (node->properties != NULL))
{
html_attr_list(rules, node->properties, parent_array, n_parent, match);
html_attr_list(rules, node->properties, parent_array, n_parent, match, mark_tag);
}
if (node->type != XML_ENTITY_REF_NODE)
@@ -397,29 +437,39 @@ static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePt
}
}
static void html_dump_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match)
static void html_dump_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
{
html_dump_one_node(rules, node, parent_array, n_parent, match);
html_dump_one_node(rules, node, parent_array, n_parent, match, mark_tag);
if(*match == 1 && mark_tag == 0)
{
return;
}
if ((node->type != XML_NAMESPACE_DECL) && (node->children != NULL) && (node->type != XML_ENTITY_REF_NODE))
{
html_node_list(rules, node->children, parent_array, n_parent, match);
html_node_list(rules, node->children, parent_array, n_parent, match, mark_tag);
}
}
static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match)
static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
{
while (node != NULL)
{
html_dump_node(rules, node, parent_array, n_parent, match);
html_dump_node(rules, node, parent_array, n_parent, match, mark_tag);
if(*match == 1 && mark_tag == 0)
{
break;
}
node = node->next;
}
}
static void html_element_foreach(const struct edit_element_rule * rules, xmlDocPtr doc, xmlNodePtr *parent_array, size_t *n_parent, int *match)
static void html_element_foreach(const struct edit_element_rule * rules, xmlDocPtr doc, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
{
if (((doc->type == XML_DOCUMENT_NODE) || (doc->type == XML_HTML_DOCUMENT_NODE)) && (doc->children != NULL))
{
html_node_list(rules, doc->children, parent_array, n_parent, match);
html_node_list(rules, doc->children, parent_array, n_parent, match, mark_tag);
}
}
@@ -476,6 +526,11 @@ size_t parse_json_output_unformatted(const char * in, size_t in_sz, const struct
}
}
if(match == 0)
{
goto finish;
}
new_out = cJSON_PrintUnformatted(interator);
if(new_out!=NULL)
{
@@ -666,9 +721,8 @@ size_t format_input_html(const char * in, size_t in_sz, const struct edit_elemen
goto finish;
}
/*When the node has inclusion relation, libxml2 is not null when deleted
So multiple loops delete **/
html_element_foreach(rules, doc, parent_array, &n_parent, &match);
/*Delete all at once, valgrind is invalid read **/
html_element_foreach(rules, doc, parent_array, &n_parent, &match, 1);
if(match != 1)
{
goto finish;
@@ -677,13 +731,15 @@ size_t format_input_html(const char * in, size_t in_sz, const struct edit_elemen
n_parent_peer = n_parent;
element_treatment=rules->element_treatment;
/*When the node has inclusion relation, libxml2 is not null when deleted
So multiple loops delete **/
if(element_treatment != NULL && !strcasecmp(element_treatment, "remove"))
{
for(i=0; i < (int)n_parent_peer; i++)
{
match =0; n_parent = 0;
html_element_foreach(rules, doc, parent_array, &n_parent, &match);
if(match == 1)
html_element_foreach(rules, doc, parent_array, &n_parent, &match, 0);
if(match == 1 && n_parent > 0)
{
xmlUnlinkNode(parent_array[0]);
xmlFreeNode(parent_array[0]);
@@ -721,7 +777,7 @@ size_t format_html_file_type(const char * interator, size_t interator_sz, const
{
size_t output_size=0;
if(interator[0] == '{')
if((interator[0] == '{') || (interator[0] == '['))
{
output_size = format_multidelete_json_type(interator, interator_sz, rule, new_out);
}

29
plugin/business/pangu-http/src/pangu_http.cpp
View File

@@ -2216,6 +2216,29 @@ static void http_manipulate(const struct tfe_stream * stream, const struct tfe_h
return;
}
static int get_fqdn_len(char *str_host)
{
char *p=NULL; int fqdn_len=0;
if(str_host == NULL)
{
goto finish;
}
p=index(str_host, ':');
if(p==NULL)
{
fqdn_len=strlen(str_host);
}
else
{
fqdn_len=p-str_host;
}
finish:
return fqdn_len;
}
enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_http_event events,
const unsigned char * body_frag, size_t frag_size, struct pangu_http_ctx * ctx, const struct tfe_stream * stream)
{
@@ -2230,10 +2253,10 @@ enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_ht
if (events & EV_HTTP_REQ_HDR)
{
const char *str_host = session->req->req_spec.host;
if (str_host != NULL)
char *str_host = (char *)session->req->req_spec.host;
int str_host_length = get_fqdn_len(str_host);
if (str_host != NULL && str_host_length != 0)
{
int str_host_length = (int) (strlen(session->req->req_spec.host));
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_HTTP_FQDN],
CHARSET_UTF8, str_host, str_host_length, result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid), ctx->thread_id);
if (scan_ret > 0)

3
plugin/business/pangu-http/src/test_edit_element.cpp
View File

@@ -101,7 +101,7 @@ TEST(EditElement, Cjson_Whole_mark_Simple)
free(output);
}
TEST(EditElement, Libxml_Whole_Remove_Facebook)
TEST(EditElement, Libxml_Inside_Remove_Facebook)
{
char* output=NULL;
size_t output_sz=0,input_len=0;
@@ -130,6 +130,7 @@ TEST(EditElement, Libxml_Whole_Remove_Facebook)
EXPECT_TRUE(output_sz>0);
EXPECT_TRUE(NULL==strstr(output, "_2t-a _4pmj _2t-d"));
free(input);
free(output);
}