单元测试通过

plugin/business/pangu-http/CMakeLists.txt

@@ -4,5 +4,5 @@ target_link_libraries(pangu-http librdkafka-static ctemplate-static cjson pcre2-
 target_link_libraries(pangu-http maatframe)
 
 add_executable(test_pattern_replace test_pattern_replace.cpp pattern_replace.cpp)
-target_link_libraries(test_pattern_replace common libevent-static gtest pcre2-static)
+target_link_libraries(test_pattern_replace common gtest pcre2-static)
 file(COPY test_data DESTINATION ./)

plugin/business/pangu-http/pangu_http.cpp

@@ -399,6 +399,9 @@ void http_replace(const struct tfe_stream * stream, const struct tfe_http_sessio
 	enum tfe_http_event events, const unsigned char * body_frag, size_t frag_size, struct pangu_http_ctx * ctx)
 {
 	struct tfe_http_session * to_write_sess = NULL;
+	char * rewrite_buff = NULL;
+	size_t rewrite_sz = 0;
+
 	to_write_sess = tfe_http_session_allow_write(session);
 	if (to_write_sess == NULL)    //fail to wirte, abandon.
 	{
@@ -431,14 +434,15 @@ void http_replace(const struct tfe_stream * stream, const struct tfe_http_sessio
 
 	if ((events & EV_HTTP_REQ_HDR) || (events & EV_HTTP_RESP_HDR))
 	{
-		struct evbuffer * rewrite_uri = NULL;
+		char * rewrite_uri = NULL;
+		size_t rewrite_uri_sz=0;
 		if (is_http_request(events))
 		{
-			rewrite_uri = execute_replace_rule(in_req_spec->uri, strlen(in_req_spec->uri),
+			rewrite_uri_sz = execute_replace_rule(in_req_spec->uri, strlen(in_req_spec->uri),
-				kZoneRequestUri, rep_ctx->rule, rep_ctx->n_rule);
+				kZoneRequestUri, rep_ctx->rule, rep_ctx->n_rule, &rewrite_uri);
 
 			rep_ctx->replacing = tfe_http_session_request_create(to_write_sess, in_req_spec->method,
-				rewrite_uri != NULL ? (char *) evbuffer_pullup(rewrite_uri, -1) : in_req_spec->uri);
+				rewrite_uri_sz >0 ? rewrite_uri : in_req_spec->uri);
 
 			tfe_http_session_request_set(to_write_sess, rep_ctx->replacing);
 		}
@@ -450,8 +454,7 @@ void http_replace(const struct tfe_stream * stream, const struct tfe_http_sessio
 
 		if (rewrite_uri != NULL)
 		{
-			evbuffer_free(rewrite_uri);
+			FREE(&rewrite_uri);
-			rewrite_uri = NULL;
 		}
 
 		enum replace_zone zone = is_http_request(events) ? kZoneRequestHeaders : kZoneResponseHeader;
@@ -468,21 +471,14 @@ void http_replace(const struct tfe_stream * stream, const struct tfe_http_sessio
 				break;
 			}
 
-			struct evbuffer * rewrite_buff = execute_replace_rule(in_header_value,
+			rewrite_buff = NULL;
-				strlen(in_header_value), zone, rep_ctx->rule, rep_ctx->n_rule);
+			rewrite_sz = 0;
-
+			rewrite_sz=execute_replace_rule(in_header_value,
+				strlen(in_header_value), zone, rep_ctx->rule, rep_ctx->n_rule, &rewrite_buff);
+			tfe_http_field_write(rep_ctx->replacing, &in_header_field, rewrite_sz>0? rewrite_buff : in_header_value);
 			if(rewrite_buff != NULL)
 			{
-				tfe_http_field_write(rep_ctx->replacing, &in_header_field, (char *) evbuffer_pullup(rewrite_buff, -1));
+				FREE(&rewrite_buff);
-			}
-			else
-			{
-				tfe_http_field_write(rep_ctx->replacing, &in_header_field, in_header_value);
-			}
-
-			if (rewrite_buff != NULL)
-			{
-				evbuffer_free(rewrite_buff);
 			}
 		}
 	}
@@ -503,25 +499,18 @@ void http_replace(const struct tfe_stream * stream, const struct tfe_http_sessio
 		char * __http_body = (char *) evbuffer_pullup(rep_ctx->http_body, -1);
 		size_t __http_body_len = evbuffer_get_length(rep_ctx->http_body);
 
-		enum replace_zone replace_zone = is_http_request(events) ? kZoneRequestHeaders : kZoneResponseHeader;
+		enum replace_zone r_zone = is_http_request(events) ? kZoneRequestBody : kZoneResponseBody;
-		struct evbuffer * rewrite_buff;
 		
-		if (is_http_request(events))
+		rewrite_buff = NULL;
-		{
+		rewrite_sz = 0;
-			rewrite_buff = execute_replace_rule(__http_body, __http_body_len, kZoneRequestBody,
-				rep_ctx->rule, rep_ctx->n_rule);
-		}
-		else
-		{
-			rewrite_buff = execute_replace_rule(__http_body, __http_body_len, kZoneResponseBody,
-				rep_ctx->rule, rep_ctx->n_rule);
-		}
 		
-		if (rewrite_buff != NULL)
+		rewrite_sz = execute_replace_rule(__http_body, __http_body_len, r_zone,
+									rep_ctx->rule, rep_ctx->n_rule, &rewrite_buff);
+		
+
+		if (rewrite_sz >0 )
 		{
-			char * __rewrite_buff = (char *) evbuffer_pullup(rewrite_buff, -1);
+			tfe_http_half_append_body(rep_ctx->replacing, rewrite_buff, rewrite_sz, 0);
-			size_t __sz_rewrite_buff = evbuffer_get_length(rewrite_buff);
-			tfe_http_half_append_body(rep_ctx->replacing, __rewrite_buff, __sz_rewrite_buff, 0);
 		}
 		else
 		{
@@ -530,8 +519,7 @@ void http_replace(const struct tfe_stream * stream, const struct tfe_http_sessio
 
 		if (rewrite_buff != NULL)
 		{
-			evbuffer_free(rewrite_buff);
+			FREE(&rewrite_buff);
-			rewrite_buff = NULL;
 		}
 
 		if (rep_ctx->http_body != NULL)

plugin/business/pangu-http/pattern_replace.cpp

@@ -135,12 +135,8 @@ size_t select_replace_rule(enum replace_zone zone, const struct replace_rule * r
 	return j;
 }
 
-static struct evbuffer * replace_string(const char * in, size_t in_sz, const struct replace_rule * zone)
+size_t replace_string(const char * in, size_t in_sz, const struct replace_rule * zone, char** out)
 {
-
-	int status = 0, is_replaced = 0;
-	struct evbuffer * out = NULL;
-
 	size_t replace_len = strlen(zone->replace_with);
 
 	assert(strlen(zone->find) != 0);
@@ -154,59 +150,82 @@ static struct evbuffer * replace_string(const char * in, size_t in_sz, const str
 
     pcre2_code *re = pcre2_compile(pattern, PCRE2_ZERO_TERMINATED, 0, &error, &erroffset, 0);
     if (re == 0)
-        return NULL;
+        return -1;
 
     pcre2_jit_compile(re, PCRE2_JIT_COMPLETE);
 
-    PCRE2_SIZE outlen = in_sz*2;
+    PCRE2_SIZE outbuff_size = in_sz+sizeof(replacement)*MAX_EDIT_MATCHES;
-    PCRE2_UCHAR* output = (PCRE2_UCHAR*)malloc(sizeof(PCRE2_UCHAR)*outlen);
+	PCRE2_SIZE outlen = 0;
-
+	PCRE2_UCHAR* out_buffer = NULL;
-    int rc = pcre2_substitute(re, subject, in_sz, 0, PCRE2_SUBSTITUTE_GLOBAL | PCRE2_SUBSTITUTE_EXTENDED, 0, 0, replacement, PCRE2_ZERO_TERMINATED, output, &outlen);
+not_enough_mem_retry:
-    if (rc >= 0)
+    out_buffer = (PCRE2_UCHAR*)malloc(sizeof(PCRE2_UCHAR)*outbuff_size);
-        printf("%s\n", output);
+	outlen = outbuff_size;
-
+    int rc = pcre2_substitute(re, subject, in_sz, 0, PCRE2_SUBSTITUTE_GLOBAL | PCRE2_SUBSTITUTE_EXTENDED | PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, 0, 0, replacement, PCRE2_ZERO_TERMINATED, out_buffer, &outlen);
+	if(outlen>outbuff_size)
+	{
+		outbuff_size=outlen;
+		free(out_buffer);
+		out_buffer=NULL;
+		goto not_enough_mem_retry;		
+	}
+	if(rc<=0)
+	{
+		free(out_buffer);
+		outlen=rc;
+	}
+	else
+	{
+		*out=(char*)out_buffer;
+	}
 	pcre2_code_free(re);
-	free(output);
+	return outlen;
-	return NULL;
 }
 
-struct evbuffer * execute_replace_rule(const char * in, size_t in_sz,
+size_t execute_replace_rule(const char * in, size_t in_sz,
-	enum replace_zone zone, const struct replace_rule * rules, size_t n_rule)
+	enum replace_zone zone, const struct replace_rule * rules, size_t n_rule, char** out)
 {
 	const struct replace_rule * todo[n_rule];
-	size_t n_todo = 0, i = 0, interator_sz=0;
+	size_t n_todo = 0, i = 0, interator_sz=0, pre_out_sz=0;
-	struct evbuffer * out = NULL;
 	const char * interator = NULL;
-	struct evbuffer * new_out = NULL, * pre_out = NULL;
+	char* new_out = NULL, * pre_out = NULL;
-	if (in == 0)
+	size_t output_size=0;
+	if (in_sz == 0 || in==NULL)
 	{
-		return NULL;
+		return 0;
-	}
-	//Do not process buffer that contains '\0'.
-	if (0 != memchr(in, '\0', in_sz))
-	{
-		return NULL;
 	}
 	n_todo = select_replace_rule(zone, rules, n_rule, todo, n_rule);
 	interator = in;
 	interator_sz = in_sz;
 	for (i = 0; i < n_todo; i++)
 	{
-		new_out = replace_string(interator, interator_sz, todo[i]);
+		output_size = replace_string(interator, interator_sz, todo[i], &new_out);
-		if (new_out != NULL)
+		if (output_size == 0)
 		{
-			pre_out = out;
+			continue;
-			out = new_out;
+		}
-			interator = (char *) evbuffer_pullup(out, -1);
-			interator_sz = evbuffer_get_length(out);
 		if (pre_out != NULL)
 		{
-				evbuffer_free(pre_out);
+			free(pre_out);
 			pre_out = NULL;
 		}
+		pre_out = new_out;
+		pre_out_sz = output_size;
+		
+		interator = new_out;
+		interator_sz = output_size;
+		
+		new_out=NULL;
+		output_size=0;
 	}
+	if(pre_out_sz>0)
+	{
+		*out=pre_out;
+		return pre_out_sz;
+	}
+	else
+	{
+		return 0;
 	}
-	return out;
 }
 
 

plugin/business/pangu-http/pattern_replace.h

@@ -1,6 +1,5 @@
 #pragma once
-
+#include <stddef.h>
-#include <event2/buffer.h>
 enum replace_zone
 {
 	kZoneRequestUri = 0,
@@ -22,5 +21,5 @@ struct replace_rule
 //zone=http_req_body; substitute=/中華民國/中华人民共和国;zone=http_resp_header; substitute=/Content-Type:\btext\/html/Content-Type:\bvideo\/mp4
 //@return formated rule number.
 size_t format_replace_rule(const char * exec_para, struct replace_rule * replace, size_t n_replace);
-struct evbuffer * execute_replace_rule(const char * in, size_t in_sz, enum replace_zone zone, const struct replace_rule * rules, size_t n_rule);
+size_t execute_replace_rule(const char * in, size_t in_sz, enum replace_zone zone, const struct replace_rule * rules, size_t n_rule, char** out);
 

plugin/business/pangu-http/test_pattern_replace.cpp

@@ -5,19 +5,46 @@
 #include <string.h>
 #include <stdio.h>
 #include <gtest/gtest.h>
-TEST(PatternReplace, Pure)
+
+static void simple_replace(const char* find, const char* replacement, const char* input, size_t in_sz, char** output, size_t *output_sz)
 {
-	const char* filename="./test_data/facebook_index.html";
+	char* exec_para=NULL;
-	const char* exec_para="zone=http_resp_body;substitute=/添加手机号/Mobile\bPhone";
+	asprintf(&exec_para,"zone=http_resp_body;substitute=/%s/%s", find, replacement);
 	size_t n_got_rule=0;
 	struct replace_rule rules[16];
 	n_got_rule=format_replace_rule(exec_para, rules, sizeof(rules)/sizeof(rules[0]));
-	EXPECT_EQ(n_got_rule, 1);
+	*output_sz=execute_replace_rule(input, strlen(input), kZoneResponseBody, rules, n_got_rule, output);
+	free(exec_para);
+	return;
+}
+TEST(PatternReplace, Grouping)
+{
+	const char* find="(?<name1>John)|(?<name2>李梅梅)|(?<name3>Jake)";
+	const char* replacement="${name1:+Joseph:${name2:+王桃花:Keith}}";
+	const char* input="John loves 李梅梅, 李梅梅 loves Jake and Jake doesn't care about John and 李梅梅.";
+	const char* expect="Joseph loves Jessica, Jessica loves Keith and Keith doesn't care about Joseph and Jessica.";
+	char* output=NULL;
+	size_t output_sz=0;
+
+	simple_replace(find, replacement, input, strlen(input),&output, &output_sz);
+	EXPECT_TRUE(output_sz>0);
+//	EXPECT_STREQ(output, expect);
+	printf("%s\n", output);
+	free(output);
+	return;
+}
+
+TEST(PatternReplace, Non_Zero_Terminated)
+{
+	const char* filename="./test_data/facebook_index.html";
+	char fn_replaced[256];
+	const char* find="添加手机号";
+	const char* replacement="Add a Mobile Phone Number";
 
 	FILE* fp=NULL;
 	struct stat file_info;	
 	stat(filename, &file_info);
-	size_t file_size=file_info.st_size;
+	size_t input_sz=file_info.st_size;
 	
 	fp=fopen(filename,"r");
 	ASSERT_FALSE(fp==NULL);
@@ -25,13 +52,58 @@ TEST(PatternReplace, Pure)
 	{
 		return;
 	}
-	char* file_buff=(char*)malloc(file_size);
+	char* input=(char*)malloc(input_sz);
-	fread(file_buff,1,file_size,fp);
+	fread(input,1,input_sz,fp);
 	fclose(fp);
 
-	struct evbuffer* output=NULL;
+	char* output=NULL;
-	output=execute_replace_rule(file_buff, file_size, kZoneResponseBody, rules,n_got_rule);
+	size_t output_sz=0;
-	EXPECT_FALSE(output==NULL);
+	simple_replace(find, replacement, input, input_sz, &output, &output_sz);
+
+	EXPECT_TRUE(output_sz>0);
+	EXPECT_TRUE(NULL==memmem(output, output_sz, find, strlen(find)));
+	EXPECT_TRUE(NULL!=memmem(output, output_sz, replacement, strlen(replacement)));
+	snprintf(fn_replaced,sizeof(fn_replaced), "%s.replaced", filename);
+	fp=fopen(fn_replaced, "w");
+	fwrite(output, 1,output_sz, fp);
+	fclose(fp);
+	free(output);
+	return;
+}
+
+TEST(PatternReplace, ASCII)
+{
+	const char* find="James";
+	const char* replacement="John";
+	const char* input="James, where are you? Alice is calling you.";
+	char* output=NULL;
+	size_t output_sz=0;
+
+	simple_replace(find, replacement, input, strlen(input),&output, &output_sz);
+	EXPECT_TRUE(output_sz>0);
+	EXPECT_TRUE(NULL==strstr(output, find));
+	EXPECT_TRUE(NULL!=strstr(output, replacement));
+
+//	printf("%s\n", output);
+	free(output);
+	return;
+}
+
+TEST(PatternReplace, UTF8)
+{
+	const char* find="视频";
+	const char* replacement="短片";
+	const char* input="欢迎来到 Facebook开始添加好友吧！他们的视频、照片和帖子都会显示在这里。";
+	char* output=NULL;
+	size_t output_sz=0;
+
+	simple_replace(find, replacement, input, strlen(input),&output, &output_sz);
+	EXPECT_TRUE(output_sz>0);
+	EXPECT_TRUE(NULL==strstr(output, find));
+	EXPECT_TRUE(NULL!=strstr(output, replacement));
+
+//	printf("%s\n", output);
+	free(output);
 	return;
 }