TSG-21649 修复client_administrative_area字段乱码问题以及Edit Element处理多json问题

common/include/tfe_http.h

@@ -300,6 +300,7 @@ typedef void (http_session_end_cb)(const struct tfe_stream * stream,
 struct http_field_name * http_field_name_duplicate(const struct http_field_name * orig);
 int http_field_name_compare(const struct http_field_name * lvalue, const struct http_field_name * rvalue);
 const char * http_field_name_to_string(const struct http_field_name * field);
+enum tfe_http_std_field http_field_name_to_std_field(const char * field_name, size_t field_name_len);
 void http_field_name_destory(struct http_field_name *);
 
 /* Tools functions for standard HTTP method */

common/src/tfe_http.cpp

@@ -110,6 +110,19 @@ struct http_field_name * http_field_construct_from_string(const char * str_field
 	return NULL;
 }
 
+enum tfe_http_std_field http_field_name_to_std_field(const char * field_name, size_t field_name_len)
+{
+	unsigned int i = 0;
+	for (i = 1; i < __str_std_header_field_map_size; i++)
+	{
+		if (field_name_len == strlen(__str_std_header_field_map[i]) && !strncasecmp(field_name, __str_std_header_field_map[i], field_name_len))
+		{
+			return (enum tfe_http_std_field)i;
+		}
+	}
+	return TFE_HTTP_UNKNOWN_FIELD;
+}
+
 const char * http_field_name_to_string(const struct http_field_name * field)
 {
 	if (field->field_id != TFE_HTTP_UNKNOWN_FIELD) return __str_std_header_field_map[field->field_id];

plugin/business/doh/src/logger.cpp

@@ -307,6 +307,7 @@ static int doh_get_ip_client_geolocation(struct tfe_cmsg * cmsg, cJSON *common_o
 
 	for(i=TFE_CMSG_SRC_REGION_STR; i <= TFE_CMSG_DST_SUBDIVISION_STR; i+=2)
 	{
+        memset(opt_val, 0, sizeof(opt_val));
 		int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)i, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
 		if (ret == 0)
 		{
@@ -325,6 +326,7 @@ static int doh_get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *common_o
 
 	for(i=TFE_CMSG_DST_REGION_STR; i <= TFE_CMSG_DST_SUBDIVISION_STR; i+=2)
 	{
+        memset(opt_val, 0, sizeof(opt_val));
 		int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)i, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
 		if (ret == 0)
 		{

plugin/business/tsg-http/src/edit_element.cpp

@@ -658,6 +658,7 @@ finish:
 
 size_t format_multidelete_json_type(const char * in, size_t in_sz, const struct edit_element_rule * rules, char** out)
 {
+	int match=0;
 	char *new_out=NULL, *pre_out=NULL;
 	char * tmp = ALLOC(char, in_sz+1);
 	char * token = NULL, * sub_token = NULL, * saveptr = NULL;
@@ -685,10 +686,25 @@ size_t format_multidelete_json_type(const char * in, size_t in_sz, const struct
 			memcpy(new_out+new_out_len, "\r\n", 2);
 			new_out_len +=2;
 			FREE(&pre_out);
+			match++;
+		}
+		else
+		{
+			memcpy(new_out+new_out_len, sub_token, strlen(sub_token));
+			new_out_len += strlen(sub_token);
+			memcpy(new_out+new_out_len, "\r\n", 2);
+			new_out_len +=2;
 		}
 	}
 
-	if(new_out)
+	if(match == 0 && new_out != NULL)
+	{
+		free(new_out);
+		new_out=NULL;
+		*out = new_out;
+		output_size = 0;
+	}
+	else if(new_out)
 	{
 		*out = new_out;
 		output_size = strlen(new_out);

plugin/business/tsg-http/src/http_lua.cpp

@@ -657,7 +657,15 @@ static int http_lua_rewrite_header(struct elua_vm *vm)
 		return 0;
 	}
 
+	enum tfe_http_std_field field_id=http_field_name_to_std_field(field_name, strlen(field_name));
+	if(field_id == TFE_HTTP_UNKNOWN_FIELD)
+	{
 		tfe_http_nonstd_field_write(tsg_ctx->replacing, field_name, field_value);
+	}
+	else
+	{
+		tfe_http_std_field_write(tsg_ctx->replacing, field_id, field_value);
+	}
 	tsg_ctx->execut_lua_sucess=1;
 	tsg_ctx->rewrite_header=1;
 

plugin/business/tsg-http/src/tsg_logger.cpp

@@ -145,6 +145,7 @@ static int get_ip_client_geolocation(struct tfe_cmsg * cmsg, cJSON *per_hit_obj)
 
 	for(i=TFE_CMSG_SRC_REGION_STR; i <= TFE_CMSG_DST_SUBDIVISION_STR; i+=2)
 	{
+		memset(opt_val, 0, sizeof(opt_val));
 		int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)i, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
 		if (ret == 0)
 		{
@@ -163,6 +164,7 @@ static int get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *per_hit_obj)
 
 	for(i=TFE_CMSG_DST_REGION_STR; i <= TFE_CMSG_DST_SUBDIVISION_STR; i+=2)
 	{
+		memset(opt_val, 0, sizeof(opt_val));
 		int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)i, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
 		if (ret == 0)
 		{

plugin/business/tsg-http/test/test_data/header_filter_by_lua.lua

@@ -0,0 +1,198 @@
+--[[
+	Request header template
+---]]
+--	content-length: LLLEEENNN
+http_resp_header = [[
+    HTTP/1.1 404 Not Found
+    Server: nqinx
+    Date: TTTIIIMMM 
+    Content-Type: text/html
+    Transfer-Encodinq: chunked
+    X-Powered-By: PHP/5.3.10
+    Set-Cookie:PHPSESSID=mmc1i020i83evg47rjb3233hb7;path=/ 
+    Expires:Thu, 19 Nov 1981 08:52:00 GMT
+    Cache-Control:no-store,no-cachemust-revalidate,post-check=0preC]
+    Pragma:no-cache
+    Location:http://UUURRRLLLUUURRRIII
+]]
+
+--[[
+	Private key template
+---]]
+
+private_key =[[
+	MIIEpAIBAAKCAQEAyAfT3h
+]]
+
+--[[
+	Response body template
+---]]
+
+http_resp_body = [[
+	<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+	<html>
+	<head>
+	<meta charset="UTF-8" http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	<title>404-对不起！您访问的页面不存在</title>
+	</head>
+	<body>
+	<div class="head404"></div>
+	<div class="txtbg404">
+	<div class="txtbox">
+	<p>对不起，您请求的页面不存在、或已被删除、或暂时不可用</p>
+	<p class="paddingbox">请点击以下链接继续浏览网页</p>
+	<p>》<a style="cursor:pointer" onclick="history.back()">返回上一页面</a></p>
+	</div>
+	</div>
+	</body>
+	</html>
+	</html>
+]]
+
+--local down_server_ip="127.0.0.1"
+function string:split(sep)
+    local sep, fields = sep or ":", {}
+    local pattern = string.format("([^%s]+)", sep)
+    self:gsub(pattern, function (c) fields[#fields + 1] = c end)
+    return fields
+end
+
+function trim(input)
+    return (string.gsub(input, "^%s*(.-)%s*$", "%1"))
+end
+
+function random(n, m)
+    math.randomseed(os.clock()*math.random(1000000,90000000)+math.random(1000000,90000000))
+    return math.random(n, m)
+end
+
+function randomNumber(len)
+    local rt = ""
+    for i=1,len,1 do
+        if i == 1 then
+            rt = rt..random(1,9)
+        else
+            rt = rt..random(0,9)
+        end
+    end
+    return rt
+end
+
+function md5sum(str)
+	str=str:gsub([[\"\'\`]],"\\%1"):sub(1)
+	return io.popen('echo -n"'..str..'"|md5sum'):read("*all"):sub(1,-5)
+  end
+
+function http_lua_ip2int( str )
+	local num = 0
+	if str and type(str)=="string" then
+		local o1,o2,o3,o4 = str:match("(%d+)%.(%d+)%.(%d+)%.(%d+)" )
+		num = 2^24*o1 + 2^16*o2 + 2^8*o3 + o4
+	end
+    return num
+end
+
+function http_lua_set_status_code(http_field_line)
+	local http_resp_line=http_field_line:split(" ")
+	for key, value in pairs(http_resp_line) do
+		if key==2 then
+			local resp_code=tonumber(value)
+			tfe.context.resp_code=resp_code
+			tfe.resp.set_status_code(resp_code)
+		end
+	end
+end
+
+function http_lua_execute_replace_rule(value, date, replace)
+
+	local in_field_value=trim(value)
+
+    in_field_value=string.gsub(in_field_value, "TTTIIIMMM", date)
+
+    in_field_value=string.gsub(in_field_value, "LLLEEENNN", #http_resp_body)
+
+    in_field_value=string.gsub(in_field_value, "UUURRRIII", replace)
+
+	--in_field_value=string.gsub(in_field_value, "UUURRRLLL", down_server_ip)
+
+    return in_field_value
+end
+
+function http_lua_set_headers(value)
+    local in_field_name, in_field_value
+
+    local http_headers=value:split(":")
+    local lenth = #http_headers
+
+	local replace=generate_secret_keys()
+	local date = string.format("%s %s", os.date("%a, %d %b %Y %I:%M:%S"), "GMT")
+
+    for key, value in pairs(http_headers) do
+        if key==1 then
+            in_field_name = string.gsub(value, "%s+", "")
+        end
+        if key==2 then
+            in_field_value=value
+        end
+        if key>2 then
+            in_field_value = string.format("%s%s", in_field_value, value)
+        end
+        if key==lenth then
+            in_field_value =http_lua_execute_replace_rule(in_field_value, date, replace)
+        end
+    end
+    tfe.resp.rewrite_header(in_field_name, in_field_value) 
+end
+
+function generate_secret_keys()
+	local src_addr
+	
+	local ip_addr_str = tfe.get_5tuple()
+	if ip_addr_str then
+		tfe.log_debug("5tuple",ip_addr_str)
+		local stream_tuple5=ip_addr_str:split(" ")
+		for key, value in pairs(stream_tuple5) do
+			if key==2 then
+				src_addr = value
+			end
+		end
+	end
+
+	local seed=tostring(randomNumber(8)+http_lua_ip2int(src_addr))
+	local the_last_eights = string.sub(seed, #seed - 7, -1)
+	if #the_last_eights == 0 then
+		return 0
+	end
+
+	local md5=md5sum(string.format("%s%s", the_last_eights, string.gsub(private_key, "%s+", "")));
+	local replace = string.sub(md5, #md5 - 7, -1)
+	--print(replace)
+	return replace
+end
+
+function header_filter_by_lua()
+	if(tfe.get_current_stage() == "http_resp_header")
+	then
+		local formatt = http_resp_header:split("\n")  
+		for key, value in pairs(formatt) do
+			if key==1 then
+				http_lua_set_status_code(value)
+			else
+				http_lua_set_headers(value)
+			end
+		end
+	end
+
+	if(tfe.get_current_stage() == "http_resp_body")
+	then
+		local resp_code='301-302'
+		if resp_code:find(tfe.context.resp_code) then
+
+		else
+			tfe.resp.set_body_data(http_resp_body)
+		end
+		
+	end
+end
+
+header_filter_by_lua()

plugin/business/tsg-http/test/test_edit_element.cpp

@@ -223,9 +223,28 @@ TEST(EditElement, LibXML_Inside_Mark_Match)
 	free(output);
 }
 
+TEST(EditElement, Cjson_multi_json_Facebook_Test)
+{
+	char* output=NULL;
+	size_t output_sz=0, input_sz=0;
+
+	const char *input="{\"A\":\"B\",\"C\":[\"D\", \"E\"],\"F\":{\"G\":{\"H\":\"I\"}},\"J\":{\"K\":[{\"L\":\"M\"}],\"N\":\"Q\"}}\n\n{\"data\":{\"user\":{\"id\":\"1\",\
+	\"units\":{\"edges\":[{\"eligible\":true,\"story\":{\"message\":{\"text\":\"safari 5.7.1 with System\"}}}]}}}}\n\n{\"data\":{\"user\":{\"id\":\"1\",\"units\":\
+	{\"edges\":[{\"eligible\":true,\"story\":{\"message\":{\"text\":\"Firefox 3.6 with System\\uff0c\\u7f16\\u8f91\"}}}]}}}}";
+
+	const char *user_region = "{\"rules\":[{\"anchor_element\":{\"search_scope\":\"whole_file\",\"start_indicator\":\"message\",\"contained_keyword\":\"safari 5.7.1 with System\"},\
+								\"target_element\":{\"target_distance_from_matching\":1,\"element_treatment\":\"remove\"}}]}";
+
+	input_sz = strlen(input);
+	simple_edit_element(user_region, input, input_sz, &output, &output_sz, 0);
+	EXPECT_TRUE(output_sz>0);
+	printf("output =%s\n", output);
+
+	free(output);
+}
+
 int main(int argc, char ** argv)
 {
-
 	::testing::InitGoogleTest(&argc, argv);
 	return RUN_ALL_TESTS();
 }

plugin/business/tsg-http/test/test_http_lua.cpp

@@ -134,6 +134,18 @@ const char *lua_http_field_read(const struct tfe_http_half * half, const struct
 	return http_head_field_node != NULL ? http_head_field->value : NULL;
 }
 
+static void lua_http_headers_clear()
+{
+	struct lua_http_head_field * http_head_field = NULL; 	struct lua_http_head_field * http_head_field_peer = NULL;
+	TAILQ_FOREACH_SAFE(http_head_field, &g_tsg_lua_pattern->lua_http_head_list.lua_http_field_list, next, http_head_field_peer)
+	{
+		TAILQ_REMOVE(&g_tsg_lua_pattern->lua_http_head_list.lua_http_field_list, http_head_field, next);
+		http_field_name_destory(http_head_field->field);
+		free(http_head_field->value);
+		free(http_head_field);
+	}
+}
+
 static int lua_http_default_headers_init(struct def_lua_http_headers *lua_http_head_value)
 {
 	for(size_t i=0; i < sizeof(def_lua_http_head_value)/sizeof(struct def_lua_http_headers); i++)
@@ -154,8 +166,8 @@ static int lua_http_default_headers_init(struct def_lua_http_headers *lua_http_h
 int http_lua_profile_for_test(int profile_id, struct elua_script ***elua_ctx, char **profile_msg, size_t *msg_len, int *timeout)
 {
 	size_t input_sz;
-	const char* filename="./test_data/http_session.lua";
+	const char *filename[]= {"./test_data/http_session.lua", "./test_data/header_filter_by_lua.lua"};
-	char *input= tfe_read_file(filename, &input_sz);
+	char *input= tfe_read_file(filename[profile_id], &input_sz);
 
 	*profile_msg=tfe_strdup(input);
 	*msg_len=input_sz;
@@ -341,8 +353,12 @@ TEST(TSG_LUA_SCRIPT, Req_Data)
 	ASSERT_TRUE(tsg_ctx.http_lua_body!=NULL);
 
 	char *__http_body=(char *) evbuffer_pullup(tsg_ctx.http_lua_body, -1);
-	EXPECT_STREQ(__http_body, "This is request data set req body");
+	size_t __http_body_len = evbuffer_get_length(tsg_ctx.http_lua_body);
-	printf("__http_body: %s\n",__http_body);
+	char __http_body_tmp[__http_body_len+1];
+	memset(__http_body_tmp, 0, sizeof(__http_body_tmp));
+	memcpy(__http_body_tmp, __http_body, __http_body_len);
+	EXPECT_STREQ(__http_body_tmp, "This is request data set req body");
+	printf("__http_body: %s\n",__http_body_tmp);
 
 	evbuffer_free(tsg_ctx.http_body);
 	evbuffer_free(tsg_ctx.http_lua_body);
@@ -356,7 +372,7 @@ TEST(TSG_LUA_SCRIPT, Req_Data)
 TEST(TSG_LUA_SCRIPT, Resq_Data)
 {
 	int ret=0;
-	int profile_id=3,thread_id=0;
+	int profile_id=0,thread_id=0;
 	struct tsg_script_ctx tsg_ctx;
 	memset(&tsg_ctx,0,sizeof(struct tsg_script_ctx));
 
@@ -375,8 +391,12 @@ TEST(TSG_LUA_SCRIPT, Resq_Data)
 	ASSERT_TRUE(tsg_ctx.http_lua_body!=NULL);
 
 	char *__http_body=(char *) evbuffer_pullup(tsg_ctx.http_lua_body, -1);
-	EXPECT_STREQ(__http_body, "This is response data set resp body");
+	size_t __http_body_len = evbuffer_get_length(tsg_ctx.http_lua_body);
-	printf("__http_body: %s\n",__http_body);
+	char __http_body_tmp[__http_body_len+1];
+	memset(__http_body_tmp, 0, sizeof(__http_body_tmp));
+	memcpy(__http_body_tmp, __http_body, __http_body_len);
+	EXPECT_STREQ(__http_body_tmp, "This is response data set resp body");
+	printf("__http_body_tmp: %s\n",__http_body_tmp);
 
 	evbuffer_free(tsg_ctx.http_body);
 	evbuffer_free(tsg_ctx.http_lua_body);
@@ -418,8 +438,12 @@ TEST(TSG_LUA_SCRIPT, Lua_Http_Session)
 	EXPECT_TRUE(tsg_ctx.http_lua_body!=NULL);
 
 	char *__http_body=(char *) evbuffer_pullup(tsg_ctx.http_lua_body, -1);
-	EXPECT_STREQ(__http_body, "This is response data set resp body");
+	size_t __http_body_len = evbuffer_get_length(tsg_ctx.http_lua_body);
-	printf("__http_body: %s\n",__http_body);
+	char __http_body_tmp[__http_body_len+1];
+	memset(__http_body_tmp, 0, sizeof(__http_body_tmp));
+	memcpy(__http_body_tmp, __http_body, __http_body_len);
+	EXPECT_STREQ(__http_body_tmp, "This is response data set resp body");
+	printf("__http_body: %s\n",__http_body_tmp);
 
 	evbuffer_free(tsg_ctx.http_body);
 	evbuffer_free(tsg_ctx.http_lua_body);
@@ -431,6 +455,53 @@ TEST(TSG_LUA_SCRIPT, Lua_Http_Session)
 	lua_http_session_destory((struct tfe_http_session *)tsg_ctx.session);
 }
 
+TEST(TSG_LUA_SCRIPT, Lua_Http_Header_Filter)
+{
+	int ret=0;
+	int profile_id=1,thread_id=0;
+	struct tsg_script_ctx tsg_ctx;	
+	memset(&tsg_ctx,0,sizeof(struct tsg_script_ctx));
+
+	tsg_ctx.session=lua_http_session_init();
+	tsg_ctx.events = EV_HTTP_RESP_HDR;
+
+	struct tsg_lua_script * lua_script=g_tsg_lua_pattern->lua_script;
+	lua_script->http_lua_profile = http_lua_profile_for_test;
+	tsg_ctx.elua_ctx=http_lua_ctx_new(lua_script, thread_id);
+
+	lua_http_headers_clear();
+	tsg_ctx.replacing=tsg_ctx.session->resp;
+	ret = execute_lua_script_rule(lua_script, profile_id, tsg_ctx.elua_ctx, thread_id, (void *)&tsg_ctx);
+	EXPECT_TRUE(ret==0);
+
+	const char *server_type_val=tfe_http_std_field_read(tsg_ctx.replacing, TFE_HTTP_SERVER);
+	EXPECT_TRUE(server_type_val!=NULL);
+	EXPECT_STREQ(server_type_val, "nqinx");
+	const char *location_type_val=tfe_http_std_field_read(tsg_ctx.replacing, TFE_HTTP_LOCATION);
+	EXPECT_TRUE(location_type_val!=NULL);
+	printf("location:%s\n",location_type_val);
+	const char *date_type_val=tfe_http_std_field_read(tsg_ctx.replacing, TFE_HTTP_DATE);
+	EXPECT_TRUE(date_type_val!=NULL);
+	printf("date:%s\n",date_type_val);
+
+	tsg_ctx.events = EV_HTTP_RESP_BODY_END;
+	tsg_ctx.http_body = evbuffer_new();
+	const char *user_input="This is response data";
+	evbuffer_add(tsg_ctx.http_body, user_input, strlen(user_input));
+
+	ret = execute_lua_script_rule(lua_script, profile_id, tsg_ctx.elua_ctx, thread_id, (void *)&tsg_ctx);
+	EXPECT_TRUE(ret==0);
+	EXPECT_TRUE(tsg_ctx.http_lua_body!=NULL);
+	char *__http_body=(char *) evbuffer_pullup(tsg_ctx.http_lua_body, -1);
+	EXPECT_TRUE(__http_body!=NULL);
+
+	elua_cleanup_script(g_tsg_lua_pattern->elua_ctx[thread_id]);
+	FREE(&g_tsg_lua_pattern->elua_ctx[thread_id]);
+	g_tsg_lua_pattern->elua_ctx[thread_id]=NULL;
+	http_lua_ctx_free(lua_script, thread_id, tsg_ctx.elua_ctx);
+	lua_http_session_destory((struct tfe_http_session *)tsg_ctx.session);
+}
+
 int main(int argc, char ** argv)
 {
 	struct tsg_lua_pattern *tsg_lua_pattern = ALLOC(struct tsg_lua_pattern, 1);

plugin/business/tsg-http/test/test_pattern_replace.cpp

@@ -213,7 +213,7 @@ TEST(PatternReplace, UrlReplace)
 		n_rule++;
 	}
 
-	size_t rewrite_uri_sz = execute_replace_rule(rd_url, strlen(rd_url), kZoneRequestUri, rule, n_rule+1, &rewrite_uri, 1);
+	size_t rewrite_uri_sz = execute_replace_rule(rd_url, strlen(rd_url), kZoneRequestUri, rule, n_rule, &rewrite_uri, 1);
 
 	for(i=0; i<n_rule; i++)
 	{