gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-20185 Proxy支持IP+Port组合object

Browse Source 
TSG-19907 修复笔误doH日志中多次发送decoded_as字段
TSG-19820 Protocol Field中Request Body/Response Body选择非运算时，无法命中策略
TSG-19540 修复Manipulation策略中Protocol选择UDP/ICMP时,选择非无法命中策略
TSG-19337 界面展示一致性，Proxy的Manipulate日志中的IP Protocol统一小写
TSG-19480 修复Metric中命中策略统计和Throughput的时机不一致，造成界面展示歧义
This commit is contained in:
fengweihao
2024-03-28 16:50:15 +08:00
parent a2dce7672a
commit 32226c9734
8 changed files with 108 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ci/travis.sh
View File

@@ -34,7 +34,7 @@ env | sort
# Install dependency from YUM
yum install -y mrzcpd numactl-devel zlib-devel librdkafka-devel systemd-devel
yum install -y libcjson-devel libmaatframe-devel libMESA_field_stat2-devel libfieldstat3-devel libMESA_handle_logger-devel libelua-devel
yum install -y libcjson-devel libmaatframe-devel libMESA_field_stat2-devel libfieldstat3-devel libfieldstat4-devel libMESA_handle_logger-devel libelua-devel
yum install -y libMESA_htable-devel libMESA_prof_load-devel libwiredcfg-devel libWiredLB-devel sapp-devel libbreakpad_mini-devel
yum install -y libasan
yum install -y numactl-libs # required by mrzcpd

1
common/src/tfe_resource.cpp
View File

@@ -152,7 +152,6 @@ static struct maat *create_maat_feather(const char *instance_name, const char *p
maat_options_set_deferred_load_on(opts);
}
maat_options_set_rule_effect_interval_ms(opts, effect_interval);
if (strlen(accept_path) > 0)
{
MESA_load_profile_string_def(accept_path, "maat", "ACCEPT_TAGS", accept_tags, sizeof(accept_tags), "{\"tags\":[{\"tag\":\"device_id\",\"value\":\"device_1\"}]}");

32
common/src/tfe_scan.cpp
View File

@@ -428,9 +428,15 @@ int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struc
{
hit_cnt_ip += n_hit_result;
}
scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_IP_PROTOCOL),
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_ip += n_hit_result;
}
scan_ret = maat_scan_ipv4((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v4->saddr,
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
scan_ret = maat_scan_ipv4_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v4->saddr, ntohs(sapp_addr.v4->source),
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_ip += n_hit_result;
@@ -447,8 +453,8 @@ int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struc
hit_cnt_ip += scan_ret;
}
scan_ret = maat_scan_ipv4((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v4->daddr,
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
scan_ret = maat_scan_ipv4_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v4->daddr, ntohs(sapp_addr.v4->dest),
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
if(scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_ip += n_hit_result;
@@ -483,9 +489,15 @@ int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struc
{
hit_cnt_ip += n_hit_result;
}
scan_ret = maat_scan_ipv6((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v6->saddr,
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_IP_PROTOCOL),
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_ip += n_hit_result;
}
scan_ret = maat_scan_ipv6_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v6->saddr, ntohs(sapp_addr.v6->source),
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_ip += n_hit_result;
@@ -501,9 +513,9 @@ int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struc
{
hit_cnt_ip += scan_ret;
}
scan_ret = maat_scan_ipv6((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v6->daddr,
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
scan_ret = maat_scan_ipv6_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v6->daddr, ntohs(sapp_addr.v6->dest),
result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_ip += n_hit_result;

3
plugin/business/doh/src/logger.cpp
View File

@@ -398,8 +398,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num));
tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num));
cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP");
cJSON_AddStringToObject(common_obj, "ip_protocol", "TCP");
cJSON_AddStringToObject(common_obj, "ip_protocol", "tcp");
cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str);

1
plugin/business/traffic-mirror/src/entry.cpp
View File

@@ -367,7 +367,6 @@ static struct maat* maat_feather_create_with_override(const char * instance_name
{
maat_options_set_deferred_load_on(opts);
}
maat_options_set_rule_effect_interval_ms(opts, effect_interval);
if (strlen(accept_tags) > 0)
{
maat_options_set_accept_tags(opts, accept_tags);

154
plugin/business/tsg-http/src/tsg_http.cpp
View File

@@ -1189,6 +1189,77 @@ struct proxy_http_ctx
int thread_id;
};
static inline int ctx_actually_replaced(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
ctx->param->action == MA_ACTION_REPLACE &&
ctx->rep_ctx->actually_replaced==1)
{
return 1;
}
else
{
return 0;
}
}
static inline int ctx_actually_ran_script(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
ctx->param->action == MA_ACTION_LUA_SCRIPT &&
ctx->tsg_ctx->actually_executed==1)
{
return 1;
}
else
{
return 0;
}
}
static inline int ctx_actually_inserted(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
ctx->param->action == MA_ACTION_INSERT &&
ctx->ins_ctx->actually_inserted==1)
{
return 1;
}
else
{
return 0;
}
}
static inline int ctx_actually_edited(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
ctx->param->action == MA_ACTION_ELEMENT && ctx->edit_ctx != NULL &&
ctx->edit_ctx->actually_edited==1)
{
return 1;
}
else
{
return 0;
}
}
static inline int ctx_actually_manipulate(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
(ctx->param->action == MA_ACTION_REDIRECT ||
ctx->param->action == MA_ACTION_HIJACK)&&
ctx->manipulate_replaced==1)
{
return 1;
}
else
{
return 0;
}
}
void http_repl_ctx_free(struct replace_ctx* rep_ctx)
{
if (rep_ctx->http_body)
@@ -2743,6 +2814,8 @@ enum proxy_action http_scan(const struct tfe_http_session * session, enum tfe_ht
if ((events & EV_HTTP_REQ_BODY_END) | (events & EV_HTTP_RESP_BODY_END))
{
table_id = events & EV_HTTP_REQ_BODY_END ? g_proxy_rt->scan_table_id[PXY_CTRL_HTTP_REQ_BODY] : g_proxy_rt
->scan_table_id[PXY_CTRL_HTTP_RES_BODY];
scan_ret = maat_scan_not_logic(g_proxy_rt->feather, table_id, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt,
&n_hit_result, ctx->scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
@@ -2829,7 +2902,12 @@ void enforce_control_policy(const struct tfe_stream * stream, const struct tfe_h
if(ctx->log_resp_body == NULL) ctx->log_resp_body = evbuffer_new();
evbuffer_add(ctx->log_resp_body, body_frag, frag_size);
}
proxy_send_metric_log(stream, ctx, thread_id, 1);
if((((ctx_actually_replaced(ctx)) || (ctx_actually_inserted(ctx)) || (ctx_actually_edited(ctx)) || (ctx_actually_manipulate(ctx))
|| ctx_actually_ran_script(ctx)) || ctx->action == PX_ACTION_REJECT || (ctx->action == PX_ACTION_MONIT)))
{
proxy_send_metric_log(stream, ctx, thread_id, 1);
}
return;
}
@@ -2910,80 +2988,6 @@ void proxy_on_http_begin(const struct tfe_stream *stream, const struct tfe_http_
return;
}
static inline int ctx_actually_replaced(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
ctx->param->action == MA_ACTION_REPLACE &&
ctx->rep_ctx->actually_replaced==1)
{
return 1;
}
else
{
return 0;
}
}
static inline int ctx_actually_ran_script(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
ctx->param->action == MA_ACTION_LUA_SCRIPT &&
ctx->tsg_ctx->actually_executed==1)
{
return 1;
}
else
{
return 0;
}
}
static inline int ctx_actually_inserted(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
ctx->param->action == MA_ACTION_INSERT &&
ctx->ins_ctx->actually_inserted==1)
{
return 1;
}
else
{
return 0;
}
}
static inline int ctx_actually_edited(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
ctx->param->action == MA_ACTION_ELEMENT && ctx->edit_ctx != NULL &&
ctx->edit_ctx->actually_edited==1)
{
return 1;
}
else
{
return 0;
}
}
static inline int ctx_actually_manipulate(struct proxy_http_ctx * ctx)
{
if(ctx->action == PX_ACTION_MANIPULATE &&
(ctx->param->action == MA_ACTION_REDIRECT ||
ctx->param->action == MA_ACTION_HIJACK)&&
ctx->manipulate_replaced==1)
{
return 1;
}
else
{
return 0;
}
}
void proxy_on_http_end(const struct tfe_stream * stream,
const struct tfe_http_session * session, unsigned int thread_id, void ** pme)
{

2
plugin/business/tsg-http/src/tsg_logger.cpp
View File

@@ -253,7 +253,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]);
cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP");
cJSON_AddStringToObject(common_obj, "ip_protocol", "TCP");
cJSON_AddStringToObject(common_obj, "ip_protocol", "tcp");
cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str);

7
resource/pangu/table_info.conf
View File

@@ -66,15 +66,16 @@
"table_id":5,
"table_name":"TSG_OBJ_IP",
"db_tables":["TSG_OBJ_IP_ADDR","TSG_OBJ_IP_LEARNING_ADDR"],
"table_type":"ip_plus",
"valid_column":7,
"table_type":"ip",
"valid_column":8,
"custom": {
"item_id":1,
"group_id":2,
"addr_type":3,
"addr_format":4,
"ip1":5,
"ip2":6
"ip2":6,
"port":7
}
},
{