From 32226c97345d3397b6c8ee11246ec0a07c4069b4 Mon Sep 17 00:00:00 2001 From: fengweihao Date: Thu, 28 Mar 2024 16:50:15 +0800 Subject: [PATCH] =?UTF-8?q?TSG-20185=20Proxy=E6=94=AF=E6=8C=81IP+Port?= =?UTF-8?q?=E7=BB=84=E5=90=88object=20TSG-19907=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=AC=94=E8=AF=AFdoH=E6=97=A5=E5=BF=97=E4=B8=AD=E5=A4=9A?= =?UTF-8?q?=E6=AC=A1=E5=8F=91=E9=80=81decoded=5Fas=E5=AD=97=E6=AE=B5=20TSG?= =?UTF-8?q?-19820=20Protocol=20Field=E4=B8=ADRequest=20Body/Response=20Bod?= =?UTF-8?q?y=E9=80=89=E6=8B=A9=E9=9D=9E=E8=BF=90=E7=AE=97=E6=97=B6?= =?UTF-8?q?=EF=BC=8C=E6=97=A0=E6=B3=95=E5=91=BD=E4=B8=AD=E7=AD=96=E7=95=A5?= =?UTF-8?q?=20TSG-19540=20=E4=BF=AE=E5=A4=8DManipulation=E7=AD=96=E7=95=A5?= =?UTF-8?q?=E4=B8=ADProtocol=E9=80=89=E6=8B=A9UDP/ICMP=E6=97=B6,=E9=80=89?= =?UTF-8?q?=E6=8B=A9=E9=9D=9E=E6=97=A0=E6=B3=95=E5=91=BD=E4=B8=AD=E7=AD=96?= =?UTF-8?q?=E7=95=A5=20TSG-19337=20=E7=95=8C=E9=9D=A2=E5=B1=95=E7=A4=BA?= =?UTF-8?q?=E4=B8=80=E8=87=B4=E6=80=A7=EF=BC=8CProxy=E7=9A=84Manipulate?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E4=B8=AD=E7=9A=84IP=20Protocol=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E5=B0=8F=E5=86=99=20TSG-19480=20=E4=BF=AE=E5=A4=8DMet?= =?UTF-8?q?ric=E4=B8=AD=E5=91=BD=E4=B8=AD=E7=AD=96=E7=95=A5=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1=E5=92=8CThroughput=E7=9A=84=E6=97=B6=E6=9C=BA?= =?UTF-8?q?=E4=B8=8D=E4=B8=80=E8=87=B4=EF=BC=8C=E9=80=A0=E6=88=90=E7=95=8C?= =?UTF-8?q?=E9=9D=A2=E5=B1=95=E7=A4=BA=E6=AD=A7=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ci/travis.sh | 2 +- common/src/tfe_resource.cpp | 1 - common/src/tfe_scan.cpp | 32 ++-- plugin/business/doh/src/logger.cpp | 3 +- plugin/business/traffic-mirror/src/entry.cpp | 1 - plugin/business/tsg-http/src/tsg_http.cpp | 154 ++++++++++--------- plugin/business/tsg-http/src/tsg_logger.cpp | 2 +- resource/pangu/table_info.conf | 7 +- 8 files changed, 108 insertions(+), 94 deletions(-) diff --git a/ci/travis.sh b/ci/travis.sh index 540a2dc..63c4264 100644 --- a/ci/travis.sh +++ b/ci/travis.sh @@ -34,7 +34,7 @@ env | sort # Install dependency from YUM yum install -y mrzcpd numactl-devel zlib-devel librdkafka-devel systemd-devel -yum install -y libcjson-devel libmaatframe-devel libMESA_field_stat2-devel libfieldstat3-devel libMESA_handle_logger-devel libelua-devel +yum install -y libcjson-devel libmaatframe-devel libMESA_field_stat2-devel libfieldstat3-devel libfieldstat4-devel libMESA_handle_logger-devel libelua-devel yum install -y libMESA_htable-devel libMESA_prof_load-devel libwiredcfg-devel libWiredLB-devel sapp-devel libbreakpad_mini-devel yum install -y libasan yum install -y numactl-libs # required by mrzcpd diff --git a/common/src/tfe_resource.cpp b/common/src/tfe_resource.cpp index f1626ae..f730f8e 100644 --- a/common/src/tfe_resource.cpp +++ b/common/src/tfe_resource.cpp @@ -152,7 +152,6 @@ static struct maat *create_maat_feather(const char *instance_name, const char *p maat_options_set_deferred_load_on(opts); } - maat_options_set_rule_effect_interval_ms(opts, effect_interval); if (strlen(accept_path) > 0) { MESA_load_profile_string_def(accept_path, "maat", "ACCEPT_TAGS", accept_tags, sizeof(accept_tags), "{\"tags\":[{\"tag\":\"device_id\",\"value\":\"device_1\"}]}"); diff --git a/common/src/tfe_scan.cpp b/common/src/tfe_scan.cpp index 2f22ac4..022db89 100644 --- a/common/src/tfe_scan.cpp +++ b/common/src/tfe_scan.cpp @@ -428,9 +428,15 @@ int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struc { hit_cnt_ip += n_hit_result; } + scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_IP_PROTOCOL), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + if (scan_ret == MAAT_SCAN_HIT) + { + hit_cnt_ip += n_hit_result; + } - scan_ret = maat_scan_ipv4((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v4->saddr, - result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + scan_ret = maat_scan_ipv4_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v4->saddr, ntohs(sapp_addr.v4->source), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if (scan_ret == MAAT_SCAN_HIT) { hit_cnt_ip += n_hit_result; @@ -447,8 +453,8 @@ int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struc hit_cnt_ip += scan_ret; } - scan_ret = maat_scan_ipv4((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v4->daddr, - result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + scan_ret = maat_scan_ipv4_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v4->daddr, ntohs(sapp_addr.v4->dest), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if(scan_ret == MAAT_SCAN_HIT) { hit_cnt_ip += n_hit_result; @@ -483,9 +489,15 @@ int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struc { hit_cnt_ip += n_hit_result; } - - scan_ret = maat_scan_ipv6((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v6->saddr, - result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_IP_PROTOCOL), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + if (scan_ret == MAAT_SCAN_HIT) + { + hit_cnt_ip += n_hit_result; + } + + scan_ret = maat_scan_ipv6_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v6->saddr, ntohs(sapp_addr.v6->source), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if (scan_ret == MAAT_SCAN_HIT) { hit_cnt_ip += n_hit_result; @@ -501,9 +513,9 @@ int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struc { hit_cnt_ip += scan_ret; } - - scan_ret = maat_scan_ipv6((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v6->daddr, - result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + + scan_ret = maat_scan_ipv6_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v6->daddr, ntohs(sapp_addr.v6->dest), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if (scan_ret == MAAT_SCAN_HIT) { hit_cnt_ip += n_hit_result; diff --git a/plugin/business/doh/src/logger.cpp b/plugin/business/doh/src/logger.cpp index 8dad40c..f269969 100644 --- a/plugin/business/doh/src/logger.cpp +++ b/plugin/business/doh/src/logger.cpp @@ -398,8 +398,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num)); tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num)); - cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP"); - cJSON_AddStringToObject(common_obj, "ip_protocol", "TCP"); + cJSON_AddStringToObject(common_obj, "ip_protocol", "tcp"); cJSON_AddNumberToObject(common_obj, "out_link_id", 0); cJSON_AddNumberToObject(common_obj, "in_link_id", 0); cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str); diff --git a/plugin/business/traffic-mirror/src/entry.cpp b/plugin/business/traffic-mirror/src/entry.cpp index cf00a2b..e9376bd 100644 --- a/plugin/business/traffic-mirror/src/entry.cpp +++ b/plugin/business/traffic-mirror/src/entry.cpp @@ -367,7 +367,6 @@ static struct maat* maat_feather_create_with_override(const char * instance_name { maat_options_set_deferred_load_on(opts); } - maat_options_set_rule_effect_interval_ms(opts, effect_interval); if (strlen(accept_tags) > 0) { maat_options_set_accept_tags(opts, accept_tags); diff --git a/plugin/business/tsg-http/src/tsg_http.cpp b/plugin/business/tsg-http/src/tsg_http.cpp index 42afd8c..5e6f96b 100644 --- a/plugin/business/tsg-http/src/tsg_http.cpp +++ b/plugin/business/tsg-http/src/tsg_http.cpp @@ -1189,6 +1189,77 @@ struct proxy_http_ctx int thread_id; }; +static inline int ctx_actually_replaced(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + ctx->param->action == MA_ACTION_REPLACE && + ctx->rep_ctx->actually_replaced==1) + { + return 1; + } + else + { + return 0; + } +} + +static inline int ctx_actually_ran_script(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + ctx->param->action == MA_ACTION_LUA_SCRIPT && + ctx->tsg_ctx->actually_executed==1) + { + return 1; + } + else + { + return 0; + } +} + +static inline int ctx_actually_inserted(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + ctx->param->action == MA_ACTION_INSERT && + ctx->ins_ctx->actually_inserted==1) + { + return 1; + } + else + { + return 0; + } +} + +static inline int ctx_actually_edited(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + ctx->param->action == MA_ACTION_ELEMENT && ctx->edit_ctx != NULL && + ctx->edit_ctx->actually_edited==1) + { + return 1; + } + else + { + return 0; + } +} + +static inline int ctx_actually_manipulate(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + (ctx->param->action == MA_ACTION_REDIRECT || + ctx->param->action == MA_ACTION_HIJACK)&& + ctx->manipulate_replaced==1) + { + return 1; + } + else + { + return 0; + } +} + void http_repl_ctx_free(struct replace_ctx* rep_ctx) { if (rep_ctx->http_body) @@ -2743,6 +2814,8 @@ enum proxy_action http_scan(const struct tfe_http_session * session, enum tfe_ht if ((events & EV_HTTP_REQ_BODY_END) | (events & EV_HTTP_RESP_BODY_END)) { + table_id = events & EV_HTTP_REQ_BODY_END ? g_proxy_rt->scan_table_id[PXY_CTRL_HTTP_REQ_BODY] : g_proxy_rt + ->scan_table_id[PXY_CTRL_HTTP_RES_BODY]; scan_ret = maat_scan_not_logic(g_proxy_rt->feather, table_id, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &n_hit_result, ctx->scan_mid); if (scan_ret == MAAT_SCAN_HIT) @@ -2829,7 +2902,12 @@ void enforce_control_policy(const struct tfe_stream * stream, const struct tfe_h if(ctx->log_resp_body == NULL) ctx->log_resp_body = evbuffer_new(); evbuffer_add(ctx->log_resp_body, body_frag, frag_size); } - proxy_send_metric_log(stream, ctx, thread_id, 1); + + if((((ctx_actually_replaced(ctx)) || (ctx_actually_inserted(ctx)) || (ctx_actually_edited(ctx)) || (ctx_actually_manipulate(ctx)) + || ctx_actually_ran_script(ctx)) || ctx->action == PX_ACTION_REJECT || (ctx->action == PX_ACTION_MONIT))) + { + proxy_send_metric_log(stream, ctx, thread_id, 1); + } return; } @@ -2910,80 +2988,6 @@ void proxy_on_http_begin(const struct tfe_stream *stream, const struct tfe_http_ return; } -static inline int ctx_actually_replaced(struct proxy_http_ctx * ctx) -{ - - if(ctx->action == PX_ACTION_MANIPULATE && - ctx->param->action == MA_ACTION_REPLACE && - ctx->rep_ctx->actually_replaced==1) - { - return 1; - } - else - { - return 0; - } -} - -static inline int ctx_actually_ran_script(struct proxy_http_ctx * ctx) -{ - if(ctx->action == PX_ACTION_MANIPULATE && - ctx->param->action == MA_ACTION_LUA_SCRIPT && - ctx->tsg_ctx->actually_executed==1) - { - return 1; - } - else - { - return 0; - } -} - -static inline int ctx_actually_inserted(struct proxy_http_ctx * ctx) -{ - - if(ctx->action == PX_ACTION_MANIPULATE && - ctx->param->action == MA_ACTION_INSERT && - ctx->ins_ctx->actually_inserted==1) - { - return 1; - } - else - { - return 0; - } -} - -static inline int ctx_actually_edited(struct proxy_http_ctx * ctx) -{ - - if(ctx->action == PX_ACTION_MANIPULATE && - ctx->param->action == MA_ACTION_ELEMENT && ctx->edit_ctx != NULL && - ctx->edit_ctx->actually_edited==1) - { - return 1; - } - else - { - return 0; - } -} - -static inline int ctx_actually_manipulate(struct proxy_http_ctx * ctx) -{ - if(ctx->action == PX_ACTION_MANIPULATE && - (ctx->param->action == MA_ACTION_REDIRECT || - ctx->param->action == MA_ACTION_HIJACK)&& - ctx->manipulate_replaced==1) - { - return 1; - } - else - { - return 0; - } -} - void proxy_on_http_end(const struct tfe_stream * stream, const struct tfe_http_session * session, unsigned int thread_id, void ** pme) { diff --git a/plugin/business/tsg-http/src/tsg_logger.cpp b/plugin/business/tsg-http/src/tsg_logger.cpp index 6673d83..4ae7203 100644 --- a/plugin/business/tsg-http/src/tsg_logger.cpp +++ b/plugin/business/tsg-http/src/tsg_logger.cpp @@ -253,7 +253,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]); cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP"); - cJSON_AddStringToObject(common_obj, "ip_protocol", "TCP"); + cJSON_AddStringToObject(common_obj, "ip_protocol", "tcp"); cJSON_AddNumberToObject(common_obj, "out_link_id", 0); cJSON_AddNumberToObject(common_obj, "in_link_id", 0); cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str); diff --git a/resource/pangu/table_info.conf b/resource/pangu/table_info.conf index e16534f..6b27e75 100644 --- a/resource/pangu/table_info.conf +++ b/resource/pangu/table_info.conf @@ -66,15 +66,16 @@ "table_id":5, "table_name":"TSG_OBJ_IP", "db_tables":["TSG_OBJ_IP_ADDR","TSG_OBJ_IP_LEARNING_ADDR"], - "table_type":"ip_plus", - "valid_column":7, + "table_type":"ip", + "valid_column":8, "custom": { "item_id":1, "group_id":2, "addr_type":3, "addr_format":4, "ip1":5, - "ip2":6 + "ip2":6, + "port":7 } }, {