TSG-22258 HTTP/2解析层不在扫描HTTP Method为CONNECT的请求，提供edit_element_tool工具获取HTML/JSON的层级关系

plugin/business/tsg-http/CMakeLists.txt

@@ -21,3 +21,7 @@ target_link_libraries(test_http_lua common gtest tsglua z)
 add_executable(replace_tool test/replace_tool.cpp src/pattern_replace.cpp)
 target_include_directories(replace_tool PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include)
 target_link_libraries(replace_tool common pcre2-static)
+
+add_executable(edit_element_tool test/edit_element_tool.cpp src/edit_element.cpp)
+target_include_directories(edit_element_tool PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include)
+target_link_libraries(edit_element_tool common pcre2-static libxml2-static z)

plugin/business/tsg-http/include/edit_element.h

@@ -10,6 +10,7 @@ enum search_scope
 
 struct edit_element_rule
 {
+    int debug_for_log;
     enum search_scope scope;
     int distane_from_matching;
     char * start_indicator;

plugin/business/tsg-http/src/edit_element.cpp

@@ -35,7 +35,7 @@ enum search_scope scope_name_to_id(const char * name)
 	return (enum search_scope) i;
 }
 
-int match_start_indicator(xmlNodePtr parent, char * start_indicator)
+int match_start_indicator(xmlNodePtr parent, char * start_indicator, int debug_for_log)
 {
 	if(parent->properties == NULL)
 	{
@@ -43,6 +43,7 @@ int match_start_indicator(xmlNodePtr parent, char * start_indicator)
 		{
 			return 0;
 		}
+		if(debug_for_log == 1) printf("  start indicator, parent name: %s\n", parent->name);
 		if(!strcasecmp((char *)parent->name, start_indicator))
 		{
 			return 1;
@@ -57,6 +58,7 @@ int match_start_indicator(xmlNodePtr parent, char * start_indicator)
 	}
 
 	xmlNodePtr children = properties->children;
+	if(debug_for_log == 1) printf("  start indicator, children content: %s\n",  children->content);
 	if(!strcasecmp((char *)children->content, start_indicator))
 	{
 		return 1;
@@ -130,6 +132,10 @@ int construct_cjson_by_treatment(cJSON *a_element, char **node, int *step, int *
 		{
 			if(a_element->type == cJSON_Object)
 			{
+				if(*node != NULL && rules->debug_for_log == 1)
+				{
+					printf("  start indicator: %s\n", *node);
+				}
 				if(*node != NULL && strcasecmp(*node, start_indicator) != 0)
 				{
 					return -2;
@@ -137,6 +143,10 @@ int construct_cjson_by_treatment(cJSON *a_element, char **node, int *step, int *
 			}
 			if(a_element->type == cJSON_Array)
 			{
+				if(a_element->string != NULL && rules->debug_for_log == 1)
+				{
+					printf("  start indicator: %s\n", a_element->string);
+				}
 				if(a_element->string != NULL && strcasecmp(a_element->string, start_indicator))
 				{
 					return -2;
@@ -165,6 +175,10 @@ int construct_cjson_by_treatment(cJSON *a_element, char **node, int *step, int *
 		{
 			if(a_element->type == cJSON_Object)
 			{
+				if(*node != NULL && rules->debug_for_log == 1)
+				{
+					printf("  start indicator: %s\n", *node);
+				}
 				if(*node != NULL && strcasecmp(*node, start_indicator) != 0)
 				{
 					return -2;
@@ -172,6 +186,10 @@ int construct_cjson_by_treatment(cJSON *a_element, char **node, int *step, int *
 			}
 			if(a_element->type == cJSON_Array)
 			{
+				if(a_element->string != NULL && rules->debug_for_log == 1)
+				{
+					printf("  start indicator: %s\n", a_element->string);
+				}
 				if(a_element->string != NULL && strcasecmp(a_element->string, start_indicator))
 				{
 					return -2;
@@ -231,6 +249,7 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP
 
 	char * start_indicator = rules->start_indicator;
 	const char *element_treatment=rules->element_treatment;
+	int debug_for_log = rules->debug_for_log;
 	int distane_from_matching = (rules->distane_from_matching + 1);
 
 	if(element_treatment != NULL &&  !strcasecmp(element_treatment, "mark"))
@@ -241,7 +260,7 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP
 	    {
 			if(k == distane_from_matching)
 			{
-				if (rules->scope == kScopeInside && match_start_indicator(parent, start_indicator) == 0)
+				if (rules->scope == kScopeInside && match_start_indicator(parent, start_indicator, debug_for_log) == 0)
 				{
 					break;
 				}
@@ -263,7 +282,7 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP
 	    {
 			if(k == distane_from_matching)
 			{
-				if (rules->scope == kScopeInside && match_start_indicator(parent, start_indicator) == 0)
+				if (rules->scope == kScopeInside && match_start_indicator(parent, start_indicator, debug_for_log) == 0)
 				{
 					break;
 				}

plugin/business/tsg-http/src/tsg_http.cpp

@@ -2937,6 +2937,12 @@ void proxy_on_http_begin(const struct tfe_stream *stream, const struct tfe_http_
 		hit_cnt += scan_ret;
 	}
 	
+	scan_ret = tfe_scan_device(stream, result, ctx->scan_mid, hit_cnt, g_proxy_rt->local_logger);
+	if(scan_ret > 0)
+	{
+		hit_cnt += scan_ret;
+	}
+
 	addr_tfe2sapp(stream->addr, &sapp_addr);
 	if (sapp_addr.addrtype == ADDR_TYPE_IPV4)
 	{

plugin/business/tsg-http/test/edit_element_tool.cpp

@@ -0,0 +1,100 @@
+#include <sys/types.h>//fstat
+#include <sys/ioctl.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+
+#include <sys/stat.h>
+#include "edit_element.h"
+
+void print_help(void)
+{
+	printf("Edit element help\n");
+	printf("-i Input File Name\n");
+	printf("-f Find KeyWords\n");
+	printf("-d Distance From Matching\n");
+	printf("-t File Content Type, 0: html 1: json\n");
+}
+
+char *edit_element_read_file(const char *filename, size_t *input_sz)
+{
+	struct stat file_info;
+	stat(filename, &file_info);
+	*input_sz=file_info.st_size;
+
+	FILE* fp=fopen(filename,"r");
+	char* input=(char*)malloc(*input_sz);
+	fread(input,1,*input_sz,fp);
+	fclose(fp);
+
+	return input;
+}
+
+int main(int argc, char ** argv)
+{
+	int distane_from_matching = 0, option=0;
+	const char* find=NULL, *filename=NULL;
+	
+	int oc=0;
+	if(argc<3)
+	{
+		print_help();
+		return -1;
+	}
+	while((oc=getopt(argc,argv,"f:d:t:i:"))!=-1)
+	{
+		switch(oc)
+		{
+			case 'i':
+				filename=strdup(optarg);
+				break;
+			case 't':
+				option=atoi(optarg);
+				break;
+			case 'f':
+				find=strdup(optarg);
+				break;
+			case 'd':
+				distane_from_matching=atoi(optarg);
+				break;
+			default:
+				printf("Invalid parameter.\n");
+				print_help();
+				return -1;
+		}
+	}
+
+	if(distane_from_matching > 128)
+	{
+		printf("Distance From Matching supports up to 128.\n");
+		print_help();
+		return -1;
+	}
+
+	char *output=NULL, *input=NULL;
+	size_t input_sz=0, n_got_rule=0;
+
+	input=edit_element_read_file(filename, &input_sz);
+	
+	struct edit_element_rule rules[128];
+	 memset(rules, 0, sizeof(rules));
+	for(int i = 0; i <= distane_from_matching; i++)
+	{
+		rules[i].debug_for_log=1;
+		rules[i].scope=kScopeInside;
+		rules[i].distane_from_matching=i;
+		rules[i].start_indicator=(char *)"log";
+		rules[i].element_treatment=(char *)"mark";
+		rules[i].contained_keyword=(char *)find;
+
+		n_got_rule=1;
+		printf("distane_from_matching: %d\n", i);
+		execute_edit_element_rule(input, strlen(input), &rules[i], n_got_rule, &output, option);
+		free(output);	
+	}
+	free(input);
+	return 0;
+}
+
+

plugin/protocol/http2/src/http2_stream.cpp

@@ -1658,8 +1658,16 @@ static void fill_req_spec_from_handle(struct tfe_h2_half_private *half_private)
 		}
 	}
 	char *urltmp = ALLOC(char, urllen + 1);
-	if(urltmp){
+	if(urltmp)
-		sprintf(urltmp, "%s%s", (char *)req_spec->host, (char *)req_spec->uri);
+	{
+		if(req_spec->host != NULL && req_spec->uri == NULL)
+		{
+			sprintf(urltmp, "%s", (char *)req_spec->host);
+		}
+		if(req_spec->host != NULL && req_spec->uri != NULL)
+		{
+			sprintf(urltmp, "%s%s", (char *)req_spec->host, (char *)req_spec->uri);
+		}
 		req_spec->url = urltmp;
 	}
 	return;
@@ -1784,6 +1792,12 @@ static int http2_client_submit_header(struct tfe_h2_stream *h2_stream_info, int3
 	fill_req_spec_from_handle(h2_session->req);	
 	//h2_stream_info->as_client->last_sent_stream_id = MIN(h2_stream_info->as_client->last_sent_stream_id, stream_id) - 1;	
 
+	if(req->half_public.req_spec.method == TFE_HTTP_METHOD_CONNECT)
+	{
+		stream_action = ACTION_FORWARD_DATA;
+		goto finish;
+	}
+
 	req->event_cb(req, EV_HTTP_REQ_HDR, NULL, 0, req->event_cb_user);
 
 	// int googletest,  h2_stream_info->tf_stream is NULL
@@ -2415,7 +2429,9 @@ static int http2_server_on_data_chunk_recv(nghttp2_session *session, uint8_t fla
 	data = input;
 	len  = input_len;
 	/*todo post data scan**/
-	if (req->body_state == H2_READ_STATE_BEGIN){
+
+	if (req->body_state == H2_READ_STATE_BEGIN)
+	{
 		if (req->event_cb) 
 		{
 			req->event_cb(req, EV_HTTP_REQ_BODY_BEGIN, NULL, len,