TSG-9422 TSG-9554 TSG-9470 修复元素编辑MARK问题, 代理支持4级或以上定位库

common/include/tfe_cmsg.h

@@ -81,6 +81,8 @@ enum tfe_cmsg_tlv_type
     TFE_CMSG_DST_IP_LOCATION_PROVINE, // string max size 256
     TFE_CMSG_SRC_IP_LOCATION_CITY,    // string max size 256
     TFE_CMSG_DST_IP_LOCATION_CITY,    // string max size 256
+    TFE_CMSG_SRC_IP_LOCATION_SUBDIVISION,
+    TFE_CMSG_DST_IP_LOCATION_SUBDIVISION,
 
     /* SSL ja3 fingerprint */
     TFE_CMSG_SSL_CLIENT_JA3_FINGERPRINT, // string max size 32

common/src/tfe_scan.cpp

@@ -134,6 +134,8 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re
     char dst_provine[TFE_STRING_MAX] = {0};
     char src_country[TFE_STRING_MAX] = {0};
     char dst_country[TFE_STRING_MAX] = {0};
+	char src_subdivsion[TFE_STRING_MAX] = {0};
+	char dst_subdivsion[TFE_STRING_MAX] = {0};
     struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
     if (cmsg != NULL)
     {
@@ -166,14 +168,32 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re
         if (scan_ret != 0)
         {
             TFE_LOG_ERROR(logger, "fetch dst city from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
+        }
+		scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_IP_LOCATION_SUBDIVISION, (unsigned char *)src_subdivsion, sizeof(src_subdivsion), &opt_out_size);
+        if (scan_ret != 0)
+        {
+            TFE_LOG_ERROR(logger, "fetch src subdivsion from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
+        }
+        scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_IP_LOCATION_SUBDIVISION, (unsigned char *)dst_subdivsion, sizeof(dst_subdivsion), &opt_out_size);
+        if (scan_ret != 0)
+        {
+            TFE_LOG_ERROR(logger, "fetch dst subdivsion from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
         }
     }
-    TFE_LOG_DEBUG(logger, "fetch src country:%s provine:%s city:%s; dst country:%s provine:%s city:%s addr: %s", src_country, src_provine, src_city, dst_country, dst_provine, dst_city, stream->str_stream_info);
+    TFE_LOG_DEBUG(logger, "fetch src country:%s provine:%s city:%s; subdivsion:%s, dst country:%s provine:%s city:%s subdivsion:%s addr: %s", src_country, src_provine, src_city, src_subdivsion, dst_country, dst_provine, dst_city, dst_subdivsion, stream->str_stream_info);
 
     if (strlen(dst_country) || strlen(dst_city))
     {
         memset(buff, 0, sizeof(buff));
-        snprintf(buff, sizeof(buff), "%s.%s.", dst_country, dst_city);
+
+		if(strlen(dst_subdivsion))
+		{
+			snprintf(buff, sizeof(buff),"%s.%s.%s.%s.", dst_country, dst_provine, dst_city, dst_subdivsion);
+		}
+		else
+		{
+			snprintf(buff, sizeof(buff), "%s.%s.", dst_country, dst_city);
+		}
         scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_LOCATION),
                                          CHARSET_GBK, buff, strlen(buff),
                                          result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
@@ -190,13 +210,27 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re
                          buff, scan_ret, stream->str_stream_info);
         }
         memset(buff, 0, sizeof(buff));
-        snprintf(buff, sizeof(buff), "%s,%s,%s", dst_city, dst_provine, dst_country);
+		if(strlen(dst_subdivsion))
+		{
+			 snprintf(buff, sizeof(buff), "%s,%s,%s,%s", dst_country, dst_provine, dst_city, dst_subdivsion);
+		}
+		else
+		{
+			 snprintf(buff, sizeof(buff), "%s,%s,%s", dst_country, dst_provine, dst_city);
+		}
         *location_server = tfe_strdup(buff);
     }
     if (strlen(src_country) || strlen(src_city))
     {
         memset(buff, 0, sizeof(buff));
-        snprintf(buff, sizeof(buff), "%s.%s.", src_country, src_city);
+		if(strlen(src_subdivsion))
+		{
+			snprintf(buff, sizeof(buff), "%s.%s.%s.%s.", src_country, src_provine, src_city, src_subdivsion);
+		}
+		else
+		{
+			snprintf(buff, sizeof(buff), "%s.%s.", src_country, src_city);
+		}
         scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_LOCATION),
                                          CHARSET_GBK, buff, strlen(buff),
                                          result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
@@ -214,7 +248,14 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re
         }
 
         memset(buff, 0, sizeof(buff));
-        snprintf(buff, sizeof(buff), "%s,%s,%s", src_city, src_provine, src_country);
+		if(strlen(src_subdivsion))
+		{
+			snprintf(buff, sizeof(buff), "%s,%s,%s,%s", src_country, src_provine, src_city, src_subdivsion);
+		}
+		else
+		{
+			snprintf(buff, sizeof(buff), "%s,%s,%s", src_country, src_provine, src_city);
+		}
         *location_client = tfe_strdup(buff);
     }
 

plugin/business/pangu-http/src/edit_element.cpp

@@ -5,6 +5,11 @@
 #include <stdlib.h>
 #include <assert.h>
 
+#if 0
+#define PCRE2_CODE_UNIT_WIDTH 8
+#include <pcre2.h>
+#endif
+
 #include <libxml/tree.h>
 #include <libxml/xmlsave.h>
 #include <libxml/HTMLparser.h>
@@ -13,7 +18,7 @@
 #include "edit_element.h"
 
 int cjson_element_foreach(cJSON *a, int *depth, int *step, int *step_level, char **node, const struct edit_element_rule * rules, int *match_num, int loop);
-static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match);
+static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match,size_t mark_tag);
 size_t parse_json_output_unformatted(const char * in, size_t in_sz, const struct edit_element_rule * rules, char** out);
 
 enum search_scope scope_name_to_id(const char * name)
@@ -52,6 +57,36 @@ int match_start_indicator(xmlNodePtr parent, char * start_indicator)
 	return 0;
 }
 
+#if 0
+int match_string(const char * in, size_t in_sz, char *contained_keyword)
+{
+	assert(strlen(contained_keyword) != 0);
+
+	int error=0;
+    PCRE2_SIZE erroffset=0;
+
+    const PCRE2_SPTR pattern = (PCRE2_SPTR)contained_keyword;
+	uint32_t pcre2_options = PCRE2_UTF;
+
+    pcre2_code *re = pcre2_compile(pattern, strlen(contained_keyword), pcre2_options, &error, &erroffset, 0);
+	if(!re)
+	{
+		return 0;
+	}
+
+	pcre2_match_data *match_data = pcre2_match_data_create_from_pattern(re, NULL);
+    int rc = 0;
+    const PCRE2_SPTR subject = (PCRE2_SPTR)in;
+
+	rc = pcre2_match(re, subject, in_sz, 0, 0, match_data, NULL);
+
+	pcre2_match_data_free(match_data);
+	pcre2_code_free(re);
+
+	return rc;
+}
+#endif
+
 int construct_cjson_by_treatment(cJSON *a_element, char **node, int *step, int *step_array_level, const struct edit_element_rule * rules)
 {
 	const char *element_treatment=rules->element_treatment;
@@ -151,6 +186,12 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP
 		}
 	}
 
+#if 0
+	if(match_string((char *)node->content, strlen((char *)node->content), rules->contained_keyword) < 0)
+	{
+		return 0;
+	}
+#endif
 	if(strcasestr((char *)node->content, rules->contained_keyword) == NULL)
 	{
 		return 0;
@@ -201,12 +242,8 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP
 					break;
 				}
 
-				if(*n_parent < 16)
+				parent_array[0] = parent;
-				{
+				*n_parent = *n_parent+1;
-					parent_array[*n_parent] = parent;
-					*n_parent = *n_parent+1;
-				}
-
 				*match =1;
 				break;
 			}
@@ -281,6 +318,9 @@ int cjson_dump_string(cJSON *a, int *depth, const struct edit_element_rule * rul
 {
 	int xret=0;
 
+#if 0
+	if((a->valuestring != NULL) && (match_string(a->valuestring, strlen(a->valuestring), rules->contained_keyword) > 0))
+#endif
 	if((a->valuestring != NULL) && strcasestr(a->valuestring, rules->contained_keyword))
 	{
 		if(*depth != -1)
@@ -345,20 +385,20 @@ static void html_namespace_list(xmlNsPtr ns)
     }
 }
 
-static void html_attr_list(const struct edit_element_rule * rules, xmlAttrPtr attr, xmlNodePtr *parent_array, size_t *n_parent, int *match)
+static void html_attr_list(const struct edit_element_rule * rules, xmlAttrPtr attr, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
 {
     while (attr != NULL)
 	{
 		if (attr->children != NULL)
 		{
-			html_node_list(rules, attr->children, parent_array, n_parent, match);
+			html_node_list(rules, attr->children, parent_array, n_parent, match, mark_tag);
 		}
 
         attr = attr->next;
     }
 }
 
-static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match)
+static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
 {
 	switch (node->type)
 	{
@@ -385,7 +425,7 @@ static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePt
 
     if ((node->type == XML_ELEMENT_NODE) && (node->properties != NULL))
     {
-        html_attr_list(rules, node->properties, parent_array, n_parent, match);
+        html_attr_list(rules, node->properties, parent_array, n_parent, match, mark_tag);
    	}
 
     if (node->type != XML_ENTITY_REF_NODE)
@@ -397,29 +437,39 @@ static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePt
     }
 }
 
-static void html_dump_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match)
+static void html_dump_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
 {
-    html_dump_one_node(rules, node, parent_array, n_parent, match);
+    html_dump_one_node(rules, node, parent_array, n_parent, match, mark_tag);
+	if(*match == 1 && mark_tag == 0)
+	{
+		return;
+	}
+
     if ((node->type != XML_NAMESPACE_DECL) && (node->children != NULL) && (node->type != XML_ENTITY_REF_NODE))
 	{
-        html_node_list(rules, node->children, parent_array, n_parent, match);
+        html_node_list(rules, node->children, parent_array, n_parent, match, mark_tag);
     }
 }
 
-static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match)
+static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
 {
     while (node != NULL)
 	{
-        html_dump_node(rules, node, parent_array, n_parent, match);
+        html_dump_node(rules, node, parent_array, n_parent, match, mark_tag);
+		if(*match == 1 && mark_tag == 0)
+		{
+			break;
+		}
+
         node = node->next;
     }
 }
 
-static void html_element_foreach(const struct edit_element_rule * rules, xmlDocPtr doc, xmlNodePtr *parent_array, size_t *n_parent, int *match)
+static void html_element_foreach(const struct edit_element_rule * rules, xmlDocPtr doc, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag)
 {
     if (((doc->type == XML_DOCUMENT_NODE) || (doc->type == XML_HTML_DOCUMENT_NODE)) && (doc->children != NULL))
 	{
-        html_node_list(rules, doc->children, parent_array, n_parent, match);
+        html_node_list(rules, doc->children, parent_array, n_parent, match, mark_tag);
     }
 }
 
@@ -476,6 +526,11 @@ size_t parse_json_output_unformatted(const char * in, size_t in_sz, const struct
 		}
 	}
 
+	if(match == 0)
+	{
+		goto finish;
+	}
+
 	new_out = cJSON_PrintUnformatted(interator);
 	if(new_out!=NULL)
 	{
@@ -666,9 +721,8 @@ size_t format_input_html(const char * in, size_t in_sz, const struct edit_elemen
 		goto finish;
 	}
 
-	/*When the node has inclusion relation, libxml2 is not null when deleted
+	/*Delete all at once, valgrind is invalid read **/
-	  So multiple loops delete	**/
+    html_element_foreach(rules, doc, parent_array, &n_parent, &match, 1);
-    html_element_foreach(rules, doc, parent_array, &n_parent, &match);
 	if(match != 1)
 	{
 		goto finish;
@@ -677,13 +731,15 @@ size_t format_input_html(const char * in, size_t in_sz, const struct edit_elemen
 	n_parent_peer = n_parent;
 	element_treatment=rules->element_treatment;
 
+	/*When the node has inclusion relation, libxml2 is not null when deleted
+	  So multiple loops delete	**/
 	if(element_treatment != NULL &&  !strcasecmp(element_treatment, "remove"))
 	{
 		for(i=0; i < (int)n_parent_peer; i++)
 		{
 			match =0; n_parent = 0;
-			html_element_foreach(rules, doc, parent_array, &n_parent, &match);
+			html_element_foreach(rules, doc, parent_array, &n_parent, &match, 0);
-			if(match == 1)
+			if(match == 1 && n_parent > 0)
 			{
 				xmlUnlinkNode(parent_array[0]);
 				xmlFreeNode(parent_array[0]);
@@ -721,7 +777,7 @@ size_t format_html_file_type(const char * interator, size_t interator_sz, const
 {
 	size_t output_size=0;
 
-	if(interator[0] == '{')
+	if((interator[0] == '{') || (interator[0] == '['))
 	{
 		output_size = format_multidelete_json_type(interator, interator_sz, rule, new_out);
 	}

plugin/business/pangu-http/src/pangu_http.cpp

@@ -2216,6 +2216,29 @@ static void http_manipulate(const struct tfe_stream * stream, const struct tfe_h
 	return;
 }
 
+static int get_fqdn_len(char *str_host)
+{
+	char *p=NULL; int fqdn_len=0;
+
+	if(str_host == NULL)
+	{
+		goto finish;
+	}
+
+	p=index(str_host, ':');
+	if(p==NULL)
+	{
+		fqdn_len=strlen(str_host);
+	}
+	else
+	{
+		fqdn_len=p-str_host;
+	}
+
+finish:
+	return fqdn_len;
+}
+
 enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_http_event events,
 	const unsigned char * body_frag, size_t frag_size, struct pangu_http_ctx * ctx, const struct tfe_stream * stream)
 {
@@ -2230,10 +2253,10 @@ enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_ht
 
 	if (events & EV_HTTP_REQ_HDR)
 	{
-		const char *str_host = session->req->req_spec.host;
+		char *str_host = (char *)session->req->req_spec.host;
-		if (str_host != NULL)
+		int str_host_length = get_fqdn_len(str_host);
+		if (str_host != NULL && str_host_length != 0)
 		{
-			int str_host_length = (int) (strlen(session->req->req_spec.host));
 			scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_HTTP_FQDN],
 								CHARSET_UTF8, str_host, str_host_length, result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid), ctx->thread_id);
 			if (scan_ret > 0)

plugin/business/pangu-http/src/test_edit_element.cpp

@@ -101,7 +101,7 @@ TEST(EditElement, Cjson_Whole_mark_Simple)
 	free(output);
 }
 
-TEST(EditElement, Libxml_Whole_Remove_Facebook)
+TEST(EditElement, Libxml_Inside_Remove_Facebook)
 {
 	char* output=NULL;
 	size_t output_sz=0,input_len=0;
@@ -130,6 +130,7 @@ TEST(EditElement, Libxml_Whole_Remove_Facebook)
 	EXPECT_TRUE(output_sz>0);
 
 	EXPECT_TRUE(NULL==strstr(output, "_2t-a _4pmj _2t-d"));
+	free(input);
 	free(output);
 }