TSG-8550 Proxy Policy支持对网页中的元素进行编辑 Element Editing

2021-12-06 18:14:10 +08:00
parent 000ab9f991
commit 8705a6fa23
5 changed files with 923 additions and 21 deletions
--- a/plugin/business/pangu-http/src/pangu_http.cpp
+++ b/plugin/business/pangu-http/src/pangu_http.cpp
@@ -1,4 +1,5 @@
 #include "pangu_logger.h"
+#include "pangu_element_edit.h"
 #include "pattern_replace.h"
 #include "pangu_web_cache.h"

@@ -53,6 +54,7 @@ enum manipulate_action
 	MA_ACTION_REPLACE,
 	MA_ACTION_HIJACK,
 	MA_ACTION_INSERT,
+	MA_ACTION_ELEMENT,
 	__MA_ACTION_MAX
 };

@@ -120,7 +122,11 @@ struct policy_action_param
 	int status_code;

 	size_t n_rule;
-	struct replace_rule *rule;
+	struct replace_rule *repl_rule;
+
+	size_t e_rule;
+	struct element_rule *elem_rule;
+
 	pthread_mutex_t lock;
 };

@@ -308,6 +314,7 @@ static enum manipulate_action manipulate_action_str2idx(const char *action_str)
 	clue_action_map[MA_ACTION_REPLACE]= "replace";
 	clue_action_map[MA_ACTION_HIJACK]= "hijack";
 	clue_action_map[MA_ACTION_INSERT]= "insert";
+	clue_action_map[MA_ACTION_ELEMENT] = "edit_element";

 	size_t i = 0;

@@ -334,7 +341,7 @@ void policy_action_param_new(int idx, const struct Maat_rule_t* rule, const char
 		return;
 	}
 	int rule_id;
-	cJSON *json=NULL, *rules=NULL, *item=NULL;
+	cJSON *json=NULL, *rules=NULL, *item=NULL, *sub_item=NULL;
 	json=cJSON_Parse(srv_def_large);
 	if(json==NULL)
 	{
@@ -419,19 +426,19 @@ void policy_action_param_new(int idx, const struct Maat_rule_t* rule, const char
 				break;
 			}
 			rule_id = 0;
-			param->rule = ALLOC(struct replace_rule, MAX_EDIT_ZONE_NUM);
+			param->repl_rule = ALLOC(struct replace_rule, MAX_EDIT_ZONE_NUM);
 			for (item = rules->child; item != NULL; item = item->next)
 			{
 				char * search  = cJSON_GetObjectItem(item , "search_in")->valuestring;
 				if (search == NULL) break;

-				param->rule[rule_id].zone = zone_name_to_id(search);
-				if (param->rule[rule_id].zone == kZoneMax)
+				param->repl_rule[rule_id].zone = zone_name_to_id(search);
+				if (param->repl_rule[rule_id].zone == kZoneMax)
 				{
 					break;
 				}
-				param->rule[rule_id].find = tfe_strdup(cJSON_GetObjectItem(item , "find")->valuestring);
-				param->rule[rule_id].replace_with = tfe_strdup(cJSON_GetObjectItem(item , "replace_with")->valuestring);
+				param->repl_rule[rule_id].find = tfe_strdup(cJSON_GetObjectItem(item , "find")->valuestring);
+				param->repl_rule[rule_id].replace_with = tfe_strdup(cJSON_GetObjectItem(item , "replace_with")->valuestring);
 				rule_id++;
 			}
 			param->n_rule = rule_id;
@@ -470,6 +477,44 @@ void policy_action_param_new(int idx, const struct Maat_rule_t* rule, const char
 				param->enforcement_ratio = 1;
 			}
 			break;
+		case MA_ACTION_ELEMENT:
+			rules = cJSON_GetObjectItem(json, "rules");
+			if(rules == NULL)
+			{
+				break;
+			}
+			rule_id = 0;
+			param->elem_rule = ALLOC(struct element_rule, MAX_EDIT_ZONE_NUM);
+			for (item = rules->child; item != NULL; item = item->next)
+			{
+				sub_item=cJSON_GetObjectItem(item,"anchor_element");
+				if(sub_item != NULL && sub_item->type ==cJSON_Object)
+				{
+					char * search_scope  = cJSON_GetObjectItem(sub_item , "search_scope")->valuestring;
+					if (search_scope == NULL) break;
+
+					param->elem_rule[rule_id].scope = scope_name_to_id(search_scope);
+					if (param->elem_rule[rule_id].scope == KScopeMax)
+					{
+						break;
+					}
+					if(param->elem_rule[rule_id].scope == kScopeInside)
+					{
+						param->elem_rule[rule_id].start_indicator = tfe_strdup(cJSON_GetObjectItem(sub_item , "start_indicator")->valuestring);
+					}
+					param->elem_rule[rule_id].contained_keyword = tfe_strdup(cJSON_GetObjectItem(sub_item,"contained_keyword")->valuestring);
+				}
+
+				sub_item=cJSON_GetObjectItem(item,"target_element");
+				if(sub_item != NULL && sub_item->type ==cJSON_Object)
+				{
+					param->elem_rule[rule_id].distane_from_matching = cJSON_GetObjectItem(sub_item , "target_distance_from_matching")->valueint;
+					param->elem_rule[rule_id].element_treatment = tfe_strdup(cJSON_GetObjectItem(sub_item,"element_treatment")->valuestring);
+				}
+				rule_id++;
+			}
+			param->e_rule = rule_id;
+			break;
 		default: assert(0);
 			break;
 	}
@@ -497,10 +542,19 @@ void policy_action_param_free_cb(int table_id, const struct Maat_rule_t* rule, c
 	}
 	pthread_mutex_unlock(&(param->lock));
 	pthread_mutex_destroy(&(param->lock));
+	for(i=0; i<param->e_rule; i++)
+	{
+		if(param->elem_rule[i].start_indicator!=NULL)
+		{
+			FREE(&(param->elem_rule[i].start_indicator));
+		}
+		FREE(&(param->elem_rule[i].element_treatment));
+		FREE(&(param->elem_rule[i].contained_keyword));
+	}
 	for(i=0; i<param->n_rule; i++)
 	{
-		FREE(&(param->rule[i].find));
-		FREE(&(param->rule[i].replace_with));
+		FREE(&(param->repl_rule[i].find));
+		FREE(&(param->repl_rule[i].replace_with));
 	}

 	if (param->message)
@@ -933,6 +987,15 @@ struct insert_ctx
 	int actually_inserted;
 };

+struct edit_element_ctx
+{
+	struct element_rule *item;
+	size_t n_item;
+	struct tfe_http_half * editing;
+	struct evbuffer *http_body;
+	int actually_edited;
+};
+
 struct ip_data_ctx
 {
 	char *asn_client;
@@ -960,6 +1023,7 @@ struct pangu_http_ctx
 	int manipulate_replaced;
 	struct replace_ctx * rep_ctx;
 	struct insert_ctx * ins_ctx;
+	struct edit_element_ctx * edit_ctx;
 	struct ip_data_ctx ip_ctx;

 	int (* resumed_cb)(const struct tfe_stream * stream,
@@ -1007,6 +1071,17 @@ void http_ins_ctx_free(struct insert_ctx* ins_ctx)
 	return;
 }

+void http_element_ctx_free(struct edit_element_ctx *edit_ctx)
+{
+	if (edit_ctx->http_body)
+	{
+		evbuffer_free(edit_ctx->http_body);
+		edit_ctx->http_body = NULL;
+	}
+	FREE(&edit_ctx);
+	return;
+}
+
 void http_ip_ctx_free(struct ip_data_ctx *ip_ctx)
 {
 	if(ip_ctx->asn_client)
@@ -1042,6 +1117,11 @@ static void pangu_http_ctx_free(struct pangu_http_ctx * ctx)
 		http_ins_ctx_free(ctx->ins_ctx);
 		ctx->ins_ctx = NULL;
 	}
+	if(ctx->edit_ctx)
+	{
+		http_element_ctx_free(ctx->edit_ctx);
+		ctx->edit_ctx = NULL;
+	}

 	http_ip_ctx_free(&ctx->ip_ctx);
 	ctx->manipulate_replaced=0;
@@ -1351,7 +1431,7 @@ void http_replace(const struct tfe_stream * stream, const struct tfe_http_sessio
 		{
 			struct policy_action_param *param = ctx->param;
 			ctx->rep_ctx = rep_ctx = ALLOC(struct replace_ctx, 1);
-			rep_ctx->rule = param->rule;
+			rep_ctx->rule = param->repl_rule;
 			rep_ctx->n_rule = param->n_rule;
 		}
 		else
@@ -1447,8 +1527,7 @@ void http_replace(const struct tfe_stream * stream, const struct tfe_http_sessio
 		{
 			options = 1;
 		}
-		rewrite_sz = execute_replace_rule(__http_body, __http_body_len, r_zone,
-									rep_ctx->rule, rep_ctx->n_rule, &rewrite_buff, options);
+		rewrite_sz = execute_replace_rule(__http_body, __http_body_len, r_zone, rep_ctx->rule, rep_ctx->n_rule, &rewrite_buff, options);

 		if (rewrite_sz >0 )
 		{
@@ -1878,7 +1957,7 @@ static void http_insert(const struct tfe_stream * stream, const struct tfe_http_
 		}
 		else
 		{
-			TFE_STREAM_LOG_INFO(stream, "Can only setup replace on REQ/RESP headers, detached.");
+			TFE_STREAM_LOG_INFO(stream, "Can only setup insert on REQ/RESP headers, detached.");
 			ctx->action = PG_ACTION_NONE;
 			tfe_http_session_detach(session); return;
 		}
@@ -1974,6 +2053,124 @@ static void http_insert(const struct tfe_stream * stream, const struct tfe_http_
 	return;
 }

+void http_element(const struct tfe_stream * stream, const struct tfe_http_session * session, enum tfe_http_event events,
+	const unsigned char * body_frag, size_t frag_size, struct pangu_http_ctx * ctx)
+{
+	struct tfe_http_session * to_write_sess = NULL;
+	char * rewrite_buff = NULL;
+	size_t rewrite_sz = 0;
+
+	if (tfe_http_in_request(events))
+	{
+		return;
+	}
+
+	to_write_sess = tfe_http_session_allow_write(session);
+	if (to_write_sess == NULL)
+	{
+		TFE_STREAM_LOG_INFO(stream, "tfe_http_session_allow_write() %s failed.", session->req->req_spec.uri);
+		ctx->action = PG_ACTION_NONE;
+		tfe_http_session_detach(session); return;
+	}
+
+	struct edit_element_ctx * edit_ctx = ctx->edit_ctx;
+	if (ctx->edit_ctx == NULL)
+	{
+		if (events & EV_HTTP_RESP_HDR)
+		{
+			struct policy_action_param *param = ctx->param;
+			ctx->edit_ctx = edit_ctx = ALLOC(struct edit_element_ctx, 1);
+			edit_ctx->item = param->elem_rule;
+			edit_ctx->n_item = param->e_rule;
+		}
+		else
+		{
+			TFE_STREAM_LOG_INFO(stream, "Can only setup editing on RESP headers, detached.");
+			ctx->action = PG_ACTION_NONE;
+			tfe_http_session_detach(session); return;
+		}
+	}
+
+	struct tfe_http_half * in_resp_half = session->resp;
+	struct tfe_http_resp_spec * in_resp_spec = &in_resp_half->resp_spec;
+
+	if (events & EV_HTTP_RESP_HDR)
+	{
+		edit_ctx->editing= tfe_http_session_response_create(to_write_sess, in_resp_spec->resp_code);
+		tfe_http_session_response_set(to_write_sess, edit_ctx->editing);
+		struct tfe_http_half * in_half = in_resp_half;
+
+		struct http_field_name in_header_field{};
+		const char * in_header_value = NULL;
+		void * iterator = NULL;
+
+		while (true)
+		{
+			if ((in_header_value = tfe_http_field_iterate(in_half, &iterator, &in_header_field)) == NULL)
+			{
+				break;
+			}
+			tfe_http_field_write(edit_ctx->editing, &in_header_field, in_header_value);
+		}
+	}
+
+	if (events & EV_HTTP_RESP_BODY_BEGIN)
+	{
+		assert(edit_ctx->http_body == NULL);
+		edit_ctx->http_body = evbuffer_new();
+	}
+
+	if (events & EV_HTTP_RESP_BODY_CONT)
+	{
+		evbuffer_add(edit_ctx->http_body, body_frag, frag_size);
+	}
+
+	if (events & EV_HTTP_RESP_BODY_END)
+	{
+		char * __http_body = (char *) evbuffer_pullup(edit_ctx->http_body, -1);
+		size_t __http_body_len = evbuffer_get_length(edit_ctx->http_body);
+
+		rewrite_buff = NULL;
+		rewrite_sz = 0;
+
+		if(in_resp_spec->content_type != NULL && strcasestr(in_resp_spec->content_type, "text/html"))
+		{
+			rewrite_sz = execute_edit_element_rule(__http_body, __http_body_len, edit_ctx->item, edit_ctx->n_item, &rewrite_buff, 0);
+		}
+		if(in_resp_spec->content_type != NULL && strcasestr(in_resp_spec->content_type, "json"))
+		{
+			rewrite_sz = execute_edit_element_rule(__http_body, __http_body_len, edit_ctx->item, edit_ctx->n_item, &rewrite_buff, 1);
+		}
+
+		if (rewrite_sz >0 )
+		{
+			tfe_http_half_append_body(edit_ctx->editing, rewrite_buff, rewrite_sz, 0);
+			edit_ctx->actually_edited=1;
+		}
+		else
+		{
+			tfe_http_half_append_body(edit_ctx->editing, __http_body, __http_body_len, 0);
+		}
+
+		if (rewrite_buff != NULL)
+		{
+			FREE(&rewrite_buff);
+		}
+
+		if (edit_ctx->http_body != NULL)
+		{
+			evbuffer_free(edit_ctx->http_body);
+			edit_ctx->http_body = NULL;
+		}
+	}
+
+	if (events & EV_HTTP_RESP_END)
+	{
+		tfe_http_half_append_body(edit_ctx->editing, NULL, 0, 0);
+		edit_ctx->editing = NULL;
+	}
+}
+
 static void http_manipulate(const struct tfe_stream * stream, const struct tfe_http_session * session,
 	enum tfe_http_event events, const unsigned char * body_frag, size_t frag_size, struct pangu_http_ctx * ctx)
 {
@@ -2003,6 +2200,9 @@ static void http_manipulate(const struct tfe_stream * stream, const struct tfe_h
 		case MA_ACTION_INSERT:
 			http_insert(stream, session, events, body_frag, frag_size, ctx);
 			break;
+		case MA_ACTION_ELEMENT:
+			http_element(stream, session, events, body_frag, frag_size, ctx);
+			break;
 		default: assert(0);
 			break;
 	}
@@ -2512,6 +2712,21 @@ static inline int ctx_actually_inserted(struct pangu_http_ctx * ctx)
 	}
 }

+static inline int ctx_actually_edited(struct pangu_http_ctx * ctx)
+{
+
+	if(ctx->action == PG_ACTION_MANIPULATE &&
+	   ctx->param->action == MA_ACTION_ELEMENT && ctx->edit_ctx != NULL &&
+	   ctx->edit_ctx->actually_edited==1)
+	{
+		return 1;
+	}
+	else
+	{
+		return 0;
+	}
+}
+
 static inline int ctx_actually_manipulate(struct pangu_http_ctx * ctx)
 {
 	if(ctx->action == PG_ACTION_MANIPULATE &&
@@ -2570,12 +2785,9 @@ void pangu_on_http_end(const struct tfe_stream * stream,
 	}

 	if(ctx->action != PG_ACTION_NONE &&
-	   (((ctx_actually_replaced(ctx)) ||
-	   (ctx_actually_inserted(ctx)) ||
-	   (ctx_actually_manipulate(ctx))) ||
-	   (ctx->action == PG_ACTION_MONIT ||
-	   	ctx->action == PG_ACTION_REJECT ||
-	   	ctx->action == PG_ACTION_WHITELIST)))
+	   (((ctx_actually_replaced(ctx)) || (ctx_actually_inserted(ctx)) || (ctx_actually_edited(ctx)) ||
+	   (ctx_actually_manipulate(ctx))) || (ctx->action == PG_ACTION_MONIT ||
+	   	ctx->action == PG_ACTION_REJECT || ctx->action == PG_ACTION_WHITELIST)))
 	{
 		ret=pangu_send_log(g_pangu_rt->send_logger, &log_msg);
 		ATOMIC_ADD(&(g_pangu_rt->stat_val[STAT_LOG_NUM]), ret);