diff --git a/common/include/tfe_http.h b/common/include/tfe_http.h index 2b0c9de..f343ea6 100644 --- a/common/include/tfe_http.h +++ b/common/include/tfe_http.h @@ -300,6 +300,7 @@ typedef void (http_session_end_cb)(const struct tfe_stream * stream, struct http_field_name * http_field_name_duplicate(const struct http_field_name * orig); int http_field_name_compare(const struct http_field_name * lvalue, const struct http_field_name * rvalue); const char * http_field_name_to_string(const struct http_field_name * field); +enum tfe_http_std_field http_field_name_to_std_field(const char * field_name, size_t field_name_len); void http_field_name_destory(struct http_field_name *); /* Tools functions for standard HTTP method */ diff --git a/common/src/tfe_http.cpp b/common/src/tfe_http.cpp index 78d27c3..36c98d7 100644 --- a/common/src/tfe_http.cpp +++ b/common/src/tfe_http.cpp @@ -110,6 +110,19 @@ struct http_field_name * http_field_construct_from_string(const char * str_field return NULL; } +enum tfe_http_std_field http_field_name_to_std_field(const char * field_name, size_t field_name_len) +{ + unsigned int i = 0; + for (i = 1; i < __str_std_header_field_map_size; i++) + { + if (field_name_len == strlen(__str_std_header_field_map[i]) && !strncasecmp(field_name, __str_std_header_field_map[i], field_name_len)) + { + return (enum tfe_http_std_field)i; + } + } + return TFE_HTTP_UNKNOWN_FIELD; +} + const char * http_field_name_to_string(const struct http_field_name * field) { if (field->field_id != TFE_HTTP_UNKNOWN_FIELD) return __str_std_header_field_map[field->field_id]; diff --git a/plugin/business/doh/src/logger.cpp b/plugin/business/doh/src/logger.cpp index db5c0ac..a1ab494 100644 --- a/plugin/business/doh/src/logger.cpp +++ b/plugin/business/doh/src/logger.cpp @@ -307,6 +307,7 @@ static int doh_get_ip_client_geolocation(struct tfe_cmsg * cmsg, cJSON *common_o for(i=TFE_CMSG_SRC_REGION_STR; i <= TFE_CMSG_DST_SUBDIVISION_STR; i+=2) { + memset(opt_val, 0, sizeof(opt_val)); int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)i, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size); if (ret == 0) { @@ -325,6 +326,7 @@ static int doh_get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *common_o for(i=TFE_CMSG_DST_REGION_STR; i <= TFE_CMSG_DST_SUBDIVISION_STR; i+=2) { + memset(opt_val, 0, sizeof(opt_val)); int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)i, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size); if (ret == 0) { diff --git a/plugin/business/tsg-http/src/edit_element.cpp b/plugin/business/tsg-http/src/edit_element.cpp index a24b202..b77d544 100644 --- a/plugin/business/tsg-http/src/edit_element.cpp +++ b/plugin/business/tsg-http/src/edit_element.cpp @@ -658,6 +658,7 @@ finish: size_t format_multidelete_json_type(const char * in, size_t in_sz, const struct edit_element_rule * rules, char** out) { + int match=0; char *new_out=NULL, *pre_out=NULL; char * tmp = ALLOC(char, in_sz+1); char * token = NULL, * sub_token = NULL, * saveptr = NULL; @@ -685,10 +686,25 @@ size_t format_multidelete_json_type(const char * in, size_t in_sz, const struct memcpy(new_out+new_out_len, "\r\n", 2); new_out_len +=2; FREE(&pre_out); + match++; + } + else + { + memcpy(new_out+new_out_len, sub_token, strlen(sub_token)); + new_out_len += strlen(sub_token); + memcpy(new_out+new_out_len, "\r\n", 2); + new_out_len +=2; } } - if(new_out) + if(match == 0 && new_out != NULL) + { + free(new_out); + new_out=NULL; + *out = new_out; + output_size = 0; + } + else if(new_out) { *out = new_out; output_size = strlen(new_out); diff --git a/plugin/business/tsg-http/src/http_lua.cpp b/plugin/business/tsg-http/src/http_lua.cpp index 78b93ba..1832645 100644 --- a/plugin/business/tsg-http/src/http_lua.cpp +++ b/plugin/business/tsg-http/src/http_lua.cpp @@ -657,7 +657,15 @@ static int http_lua_rewrite_header(struct elua_vm *vm) return 0; } - tfe_http_nonstd_field_write(tsg_ctx->replacing, field_name, field_value); + enum tfe_http_std_field field_id=http_field_name_to_std_field(field_name, strlen(field_name)); + if(field_id == TFE_HTTP_UNKNOWN_FIELD) + { + tfe_http_nonstd_field_write(tsg_ctx->replacing, field_name, field_value); + } + else + { + tfe_http_std_field_write(tsg_ctx->replacing, field_id, field_value); + } tsg_ctx->execut_lua_sucess=1; tsg_ctx->rewrite_header=1; diff --git a/plugin/business/tsg-http/src/tsg_logger.cpp b/plugin/business/tsg-http/src/tsg_logger.cpp index 52a99e6..e1cc528 100644 --- a/plugin/business/tsg-http/src/tsg_logger.cpp +++ b/plugin/business/tsg-http/src/tsg_logger.cpp @@ -145,6 +145,7 @@ static int get_ip_client_geolocation(struct tfe_cmsg * cmsg, cJSON *per_hit_obj) for(i=TFE_CMSG_SRC_REGION_STR; i <= TFE_CMSG_DST_SUBDIVISION_STR; i+=2) { + memset(opt_val, 0, sizeof(opt_val)); int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)i, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size); if (ret == 0) { @@ -163,6 +164,7 @@ static int get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *per_hit_obj) for(i=TFE_CMSG_DST_REGION_STR; i <= TFE_CMSG_DST_SUBDIVISION_STR; i+=2) { + memset(opt_val, 0, sizeof(opt_val)); int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)i, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size); if (ret == 0) { diff --git a/plugin/business/tsg-http/test/test_data/header_filter_by_lua.lua b/plugin/business/tsg-http/test/test_data/header_filter_by_lua.lua new file mode 100644 index 0000000..a7e2f47 --- /dev/null +++ b/plugin/business/tsg-http/test/test_data/header_filter_by_lua.lua @@ -0,0 +1,198 @@ +--[[ + Request header template +---]] +-- content-length: LLLEEENNN +http_resp_header = [[ + HTTP/1.1 404 Not Found + Server: nqinx + Date: TTTIIIMMM + Content-Type: text/html + Transfer-Encodinq: chunked + X-Powered-By: PHP/5.3.10 + Set-Cookie:PHPSESSID=mmc1i020i83evg47rjb3233hb7;path=/ + Expires:Thu, 19 Nov 1981 08:52:00 GMT + Cache-Control:no-store,no-cachemust-revalidate,post-check=0preC] + Pragma:no-cache + Location:http://UUURRRLLLUUURRRIII +]] + +--[[ + Private key template +---]] + +private_key =[[ + MIIEpAIBAAKCAQEAyAfT3h +]] + +--[[ + Response body template +---]] + +http_resp_body = [[ + + + + + 404-对不起！您访问的页面不存在 + + +
+
+
+

对不起，您请求的页面不存在、或已被删除、或暂时不可用

+

请点击以下链接继续浏览网页

+

》返回上一页面

+
+
+ + + +]] + +--local down_server_ip="127.0.0.1" +function string:split(sep) + local sep, fields = sep or ":", {} + local pattern = string.format("([^%s]+)", sep) + self:gsub(pattern, function (c) fields[#fields + 1] = c end) + return fields +end + +function trim(input) + return (string.gsub(input, "^%s*(.-)%s*$", "%1")) +end + +function random(n, m) + math.randomseed(os.clock()*math.random(1000000,90000000)+math.random(1000000,90000000)) + return math.random(n, m) +end + +function randomNumber(len) + local rt = "" + for i=1,len,1 do + if i == 1 then + rt = rt..random(1,9) + else + rt = rt..random(0,9) + end + end + return rt +end + +function md5sum(str) + str=str:gsub([[\"\'\`]],"\\%1"):sub(1) + return io.popen('echo -n"'..str..'"|md5sum'):read("*all"):sub(1,-5) + end + +function http_lua_ip2int( str ) + local num = 0 + if str and type(str)=="string" then + local o1,o2,o3,o4 = str:match("(%d+)%.(%d+)%.(%d+)%.(%d+)" ) + num = 2^24*o1 + 2^16*o2 + 2^8*o3 + o4 + end + return num +end + +function http_lua_set_status_code(http_field_line) + local http_resp_line=http_field_line:split(" ") + for key, value in pairs(http_resp_line) do + if key==2 then + local resp_code=tonumber(value) + tfe.context.resp_code=resp_code + tfe.resp.set_status_code(resp_code) + end + end +end + +function http_lua_execute_replace_rule(value, date, replace) + + local in_field_value=trim(value) + + in_field_value=string.gsub(in_field_value, "TTTIIIMMM", date) + + in_field_value=string.gsub(in_field_value, "LLLEEENNN", #http_resp_body) + + in_field_value=string.gsub(in_field_value, "UUURRRIII", replace) + + --in_field_value=string.gsub(in_field_value, "UUURRRLLL", down_server_ip) + + return in_field_value +end + +function http_lua_set_headers(value) + local in_field_name, in_field_value + + local http_headers=value:split(":") + local lenth = #http_headers + + local replace=generate_secret_keys() + local date = string.format("%s %s", os.date("%a, %d %b %Y %I:%M:%S"), "GMT") + + for key, value in pairs(http_headers) do + if key==1 then + in_field_name = string.gsub(value, "%s+", "") + end + if key==2 then + in_field_value=value + end + if key>2 then + in_field_value = string.format("%s%s", in_field_value, value) + end + if key==lenth then + in_field_value =http_lua_execute_replace_rule(in_field_value, date, replace) + end + end + tfe.resp.rewrite_header(in_field_name, in_field_value) +end + +function generate_secret_keys() + local src_addr + + local ip_addr_str = tfe.get_5tuple() + if ip_addr_str then + tfe.log_debug("5tuple",ip_addr_str) + local stream_tuple5=ip_addr_str:split(" ") + for key, value in pairs(stream_tuple5) do + if key==2 then + src_addr = value + end + end + end + + local seed=tostring(randomNumber(8)+http_lua_ip2int(src_addr)) + local the_last_eights = string.sub(seed, #seed - 7, -1) + if #the_last_eights == 0 then + return 0 + end + + local md5=md5sum(string.format("%s%s", the_last_eights, string.gsub(private_key, "%s+", ""))); + local replace = string.sub(md5, #md5 - 7, -1) + --print(replace) + return replace +end + +function header_filter_by_lua() + if(tfe.get_current_stage() == "http_resp_header") + then + local formatt = http_resp_header:split("\n") + for key, value in pairs(formatt) do + if key==1 then + http_lua_set_status_code(value) + else + http_lua_set_headers(value) + end + end + end + + if(tfe.get_current_stage() == "http_resp_body") + then + local resp_code='301-302' + if resp_code:find(tfe.context.resp_code) then + + else + tfe.resp.set_body_data(http_resp_body) + end + + end +end + +header_filter_by_lua() \ No newline at end of file diff --git a/plugin/business/tsg-http/test/test_edit_element.cpp b/plugin/business/tsg-http/test/test_edit_element.cpp index 5893846..d441674 100644 --- a/plugin/business/tsg-http/test/test_edit_element.cpp +++ b/plugin/business/tsg-http/test/test_edit_element.cpp @@ -223,9 +223,28 @@ TEST(EditElement, LibXML_Inside_Mark_Match) free(output); } +TEST(EditElement, Cjson_multi_json_Facebook_Test) +{ + char* output=NULL; + size_t output_sz=0, input_sz=0; + + const char *input="{\"A\":\"B\",\"C\":[\"D\", \"E\"],\"F\":{\"G\":{\"H\":\"I\"}},\"J\":{\"K\":[{\"L\":\"M\"}],\"N\":\"Q\"}}\n\n{\"data\":{\"user\":{\"id\":\"1\",\ + \"units\":{\"edges\":[{\"eligible\":true,\"story\":{\"message\":{\"text\":\"safari 5.7.1 with System\"}}}]}}}}\n\n{\"data\":{\"user\":{\"id\":\"1\",\"units\":\ + {\"edges\":[{\"eligible\":true,\"story\":{\"message\":{\"text\":\"Firefox 3.6 with System\\uff0c\\u7f16\\u8f91\"}}}]}}}}"; + + const char *user_region = "{\"rules\":[{\"anchor_element\":{\"search_scope\":\"whole_file\",\"start_indicator\":\"message\",\"contained_keyword\":\"safari 5.7.1 with System\"},\ + \"target_element\":{\"target_distance_from_matching\":1,\"element_treatment\":\"remove\"}}]}"; + + input_sz = strlen(input); + simple_edit_element(user_region, input, input_sz, &output, &output_sz, 0); + EXPECT_TRUE(output_sz>0); + printf("output =%s\n", output); + + free(output); +} + int main(int argc, char ** argv) { - ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/plugin/business/tsg-http/test/test_http_lua.cpp b/plugin/business/tsg-http/test/test_http_lua.cpp index bf99e46..14daf52 100644 --- a/plugin/business/tsg-http/test/test_http_lua.cpp +++ b/plugin/business/tsg-http/test/test_http_lua.cpp @@ -134,6 +134,18 @@ const char *lua_http_field_read(const struct tfe_http_half * half, const struct return http_head_field_node != NULL ? http_head_field->value : NULL; } +static void lua_http_headers_clear() +{ + struct lua_http_head_field * http_head_field = NULL; struct lua_http_head_field * http_head_field_peer = NULL; + TAILQ_FOREACH_SAFE(http_head_field, &g_tsg_lua_pattern->lua_http_head_list.lua_http_field_list, next, http_head_field_peer) + { + TAILQ_REMOVE(&g_tsg_lua_pattern->lua_http_head_list.lua_http_field_list, http_head_field, next); + http_field_name_destory(http_head_field->field); + free(http_head_field->value); + free(http_head_field); + } +} + static int lua_http_default_headers_init(struct def_lua_http_headers *lua_http_head_value) { for(size_t i=0; i < sizeof(def_lua_http_head_value)/sizeof(struct def_lua_http_headers); i++) @@ -154,8 +166,8 @@ static int lua_http_default_headers_init(struct def_lua_http_headers *lua_http_h int http_lua_profile_for_test(int profile_id, struct elua_script ***elua_ctx, char **profile_msg, size_t *msg_len, int *timeout) { size_t input_sz; - const char* filename="./test_data/http_session.lua"; - char *input= tfe_read_file(filename, &input_sz); + const char *filename[]= {"./test_data/http_session.lua", "./test_data/header_filter_by_lua.lua"}; + char *input= tfe_read_file(filename[profile_id], &input_sz); *profile_msg=tfe_strdup(input); *msg_len=input_sz; @@ -341,8 +353,12 @@ TEST(TSG_LUA_SCRIPT, Req_Data) ASSERT_TRUE(tsg_ctx.http_lua_body!=NULL); char *__http_body=(char *) evbuffer_pullup(tsg_ctx.http_lua_body, -1); - EXPECT_STREQ(__http_body, "This is request data set req body"); - printf("__http_body: %s\n",__http_body); + size_t __http_body_len = evbuffer_get_length(tsg_ctx.http_lua_body); + char __http_body_tmp[__http_body_len+1]; + memset(__http_body_tmp, 0, sizeof(__http_body_tmp)); + memcpy(__http_body_tmp, __http_body, __http_body_len); + EXPECT_STREQ(__http_body_tmp, "This is request data set req body"); + printf("__http_body: %s\n",__http_body_tmp); evbuffer_free(tsg_ctx.http_body); evbuffer_free(tsg_ctx.http_lua_body); @@ -356,7 +372,7 @@ TEST(TSG_LUA_SCRIPT, Req_Data) TEST(TSG_LUA_SCRIPT, Resq_Data) { int ret=0; - int profile_id=3,thread_id=0; + int profile_id=0,thread_id=0; struct tsg_script_ctx tsg_ctx; memset(&tsg_ctx,0,sizeof(struct tsg_script_ctx)); @@ -375,8 +391,12 @@ TEST(TSG_LUA_SCRIPT, Resq_Data) ASSERT_TRUE(tsg_ctx.http_lua_body!=NULL); char *__http_body=(char *) evbuffer_pullup(tsg_ctx.http_lua_body, -1); - EXPECT_STREQ(__http_body, "This is response data set resp body"); - printf("__http_body: %s\n",__http_body); + size_t __http_body_len = evbuffer_get_length(tsg_ctx.http_lua_body); + char __http_body_tmp[__http_body_len+1]; + memset(__http_body_tmp, 0, sizeof(__http_body_tmp)); + memcpy(__http_body_tmp, __http_body, __http_body_len); + EXPECT_STREQ(__http_body_tmp, "This is response data set resp body"); + printf("__http_body_tmp: %s\n",__http_body_tmp); evbuffer_free(tsg_ctx.http_body); evbuffer_free(tsg_ctx.http_lua_body); @@ -418,8 +438,12 @@ TEST(TSG_LUA_SCRIPT, Lua_Http_Session) EXPECT_TRUE(tsg_ctx.http_lua_body!=NULL); char *__http_body=(char *) evbuffer_pullup(tsg_ctx.http_lua_body, -1); - EXPECT_STREQ(__http_body, "This is response data set resp body"); - printf("__http_body: %s\n",__http_body); + size_t __http_body_len = evbuffer_get_length(tsg_ctx.http_lua_body); + char __http_body_tmp[__http_body_len+1]; + memset(__http_body_tmp, 0, sizeof(__http_body_tmp)); + memcpy(__http_body_tmp, __http_body, __http_body_len); + EXPECT_STREQ(__http_body_tmp, "This is response data set resp body"); + printf("__http_body: %s\n",__http_body_tmp); evbuffer_free(tsg_ctx.http_body); evbuffer_free(tsg_ctx.http_lua_body); @@ -431,6 +455,53 @@ TEST(TSG_LUA_SCRIPT, Lua_Http_Session) lua_http_session_destory((struct tfe_http_session *)tsg_ctx.session); } +TEST(TSG_LUA_SCRIPT, Lua_Http_Header_Filter) +{ + int ret=0; + int profile_id=1,thread_id=0; + struct tsg_script_ctx tsg_ctx; + memset(&tsg_ctx,0,sizeof(struct tsg_script_ctx)); + + tsg_ctx.session=lua_http_session_init(); + tsg_ctx.events = EV_HTTP_RESP_HDR; + + struct tsg_lua_script * lua_script=g_tsg_lua_pattern->lua_script; + lua_script->http_lua_profile = http_lua_profile_for_test; + tsg_ctx.elua_ctx=http_lua_ctx_new(lua_script, thread_id); + + lua_http_headers_clear(); + tsg_ctx.replacing=tsg_ctx.session->resp; + ret = execute_lua_script_rule(lua_script, profile_id, tsg_ctx.elua_ctx, thread_id, (void *)&tsg_ctx); + EXPECT_TRUE(ret==0); + + const char *server_type_val=tfe_http_std_field_read(tsg_ctx.replacing, TFE_HTTP_SERVER); + EXPECT_TRUE(server_type_val!=NULL); + EXPECT_STREQ(server_type_val, "nqinx"); + const char *location_type_val=tfe_http_std_field_read(tsg_ctx.replacing, TFE_HTTP_LOCATION); + EXPECT_TRUE(location_type_val!=NULL); + printf("location:%s\n",location_type_val); + const char *date_type_val=tfe_http_std_field_read(tsg_ctx.replacing, TFE_HTTP_DATE); + EXPECT_TRUE(date_type_val!=NULL); + printf("date:%s\n",date_type_val); + + tsg_ctx.events = EV_HTTP_RESP_BODY_END; + tsg_ctx.http_body = evbuffer_new(); + const char *user_input="This is response data"; + evbuffer_add(tsg_ctx.http_body, user_input, strlen(user_input)); + + ret = execute_lua_script_rule(lua_script, profile_id, tsg_ctx.elua_ctx, thread_id, (void *)&tsg_ctx); + EXPECT_TRUE(ret==0); + EXPECT_TRUE(tsg_ctx.http_lua_body!=NULL); + char *__http_body=(char *) evbuffer_pullup(tsg_ctx.http_lua_body, -1); + EXPECT_TRUE(__http_body!=NULL); + + elua_cleanup_script(g_tsg_lua_pattern->elua_ctx[thread_id]); + FREE(&g_tsg_lua_pattern->elua_ctx[thread_id]); + g_tsg_lua_pattern->elua_ctx[thread_id]=NULL; + http_lua_ctx_free(lua_script, thread_id, tsg_ctx.elua_ctx); + lua_http_session_destory((struct tfe_http_session *)tsg_ctx.session); +} + int main(int argc, char ** argv) { struct tsg_lua_pattern *tsg_lua_pattern = ALLOC(struct tsg_lua_pattern, 1); diff --git a/plugin/business/tsg-http/test/test_pattern_replace.cpp b/plugin/business/tsg-http/test/test_pattern_replace.cpp index 9df3e88..50fd239 100644 --- a/plugin/business/tsg-http/test/test_pattern_replace.cpp +++ b/plugin/business/tsg-http/test/test_pattern_replace.cpp @@ -213,7 +213,7 @@ TEST(PatternReplace, UrlReplace) n_rule++; } - size_t rewrite_uri_sz = execute_replace_rule(rd_url, strlen(rd_url), kZoneRequestUri, rule, n_rule+1, &rewrite_uri, 1); + size_t rewrite_uri_sz = execute_replace_rule(rd_url, strlen(rd_url), kZoneRequestUri, rule, n_rule, &rewrite_uri, 1); for(i=0; i