diff --git a/ci/travis.sh b/ci/travis.sh index 540a2dc..63c4264 100644 --- a/ci/travis.sh +++ b/ci/travis.sh @@ -34,7 +34,7 @@ env | sort # Install dependency from YUM yum install -y mrzcpd numactl-devel zlib-devel librdkafka-devel systemd-devel -yum install -y libcjson-devel libmaatframe-devel libMESA_field_stat2-devel libfieldstat3-devel libMESA_handle_logger-devel libelua-devel +yum install -y libcjson-devel libmaatframe-devel libMESA_field_stat2-devel libfieldstat3-devel libfieldstat4-devel libMESA_handle_logger-devel libelua-devel yum install -y libMESA_htable-devel libMESA_prof_load-devel libwiredcfg-devel libWiredLB-devel sapp-devel libbreakpad_mini-devel yum install -y libasan yum install -y numactl-libs # required by mrzcpd diff --git a/common/src/tfe_resource.cpp b/common/src/tfe_resource.cpp index f1626ae..f730f8e 100644 --- a/common/src/tfe_resource.cpp +++ b/common/src/tfe_resource.cpp @@ -152,7 +152,6 @@ static struct maat *create_maat_feather(const char *instance_name, const char *p maat_options_set_deferred_load_on(opts); } - maat_options_set_rule_effect_interval_ms(opts, effect_interval); if (strlen(accept_path) > 0) { MESA_load_profile_string_def(accept_path, "maat", "ACCEPT_TAGS", accept_tags, sizeof(accept_tags), "{\"tags\":[{\"tag\":\"device_id\",\"value\":\"device_1\"}]}"); diff --git a/common/src/tfe_scan.cpp b/common/src/tfe_scan.cpp index 2f22ac4..022db89 100644 --- a/common/src/tfe_scan.cpp +++ b/common/src/tfe_scan.cpp @@ -428,9 +428,15 @@ int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struc { hit_cnt_ip += n_hit_result; } + scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_IP_PROTOCOL), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + if (scan_ret == MAAT_SCAN_HIT) + { + hit_cnt_ip += n_hit_result; + } - scan_ret = maat_scan_ipv4((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v4->saddr, - result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + scan_ret = maat_scan_ipv4_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v4->saddr, ntohs(sapp_addr.v4->source), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if (scan_ret == MAAT_SCAN_HIT) { hit_cnt_ip += n_hit_result; @@ -447,8 +453,8 @@ int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struc hit_cnt_ip += scan_ret; } - scan_ret = maat_scan_ipv4((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v4->daddr, - result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + scan_ret = maat_scan_ipv4_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v4->daddr, ntohs(sapp_addr.v4->dest), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if(scan_ret == MAAT_SCAN_HIT) { hit_cnt_ip += n_hit_result; @@ -483,9 +489,15 @@ int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struc { hit_cnt_ip += n_hit_result; } - - scan_ret = maat_scan_ipv6((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v6->saddr, - result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + scan_ret = maat_scan_not_logic((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_IP_PROTOCOL), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + if (scan_ret == MAAT_SCAN_HIT) + { + hit_cnt_ip += n_hit_result; + } + + scan_ret = maat_scan_ipv6_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_SOURCE_IP), sapp_addr.v6->saddr, ntohs(sapp_addr.v6->source), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if (scan_ret == MAAT_SCAN_HIT) { hit_cnt_ip += n_hit_result; @@ -501,9 +513,9 @@ int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struc { hit_cnt_ip += scan_ret; } - - scan_ret = maat_scan_ipv6((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v6->daddr, - result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); + + scan_ret = maat_scan_ipv6_port((struct maat *)tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(PXY_CTRL_DESTINATION_IP), sapp_addr.v6->daddr, ntohs(sapp_addr.v6->dest), + result+hit_cnt+hit_cnt_ip, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip, &n_hit_result, scan_mid); if (scan_ret == MAAT_SCAN_HIT) { hit_cnt_ip += n_hit_result; diff --git a/plugin/business/doh/src/logger.cpp b/plugin/business/doh/src/logger.cpp index 8dad40c..f269969 100644 --- a/plugin/business/doh/src/logger.cpp +++ b/plugin/business/doh/src/logger.cpp @@ -398,8 +398,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num)); tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num)); - cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP"); - cJSON_AddStringToObject(common_obj, "ip_protocol", "TCP"); + cJSON_AddStringToObject(common_obj, "ip_protocol", "tcp"); cJSON_AddNumberToObject(common_obj, "out_link_id", 0); cJSON_AddNumberToObject(common_obj, "in_link_id", 0); cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str); diff --git a/plugin/business/traffic-mirror/src/entry.cpp b/plugin/business/traffic-mirror/src/entry.cpp index cf00a2b..e9376bd 100644 --- a/plugin/business/traffic-mirror/src/entry.cpp +++ b/plugin/business/traffic-mirror/src/entry.cpp @@ -367,7 +367,6 @@ static struct maat* maat_feather_create_with_override(const char * instance_name { maat_options_set_deferred_load_on(opts); } - maat_options_set_rule_effect_interval_ms(opts, effect_interval); if (strlen(accept_tags) > 0) { maat_options_set_accept_tags(opts, accept_tags); diff --git a/plugin/business/tsg-http/src/tsg_http.cpp b/plugin/business/tsg-http/src/tsg_http.cpp index 42afd8c..5e6f96b 100644 --- a/plugin/business/tsg-http/src/tsg_http.cpp +++ b/plugin/business/tsg-http/src/tsg_http.cpp @@ -1189,6 +1189,77 @@ struct proxy_http_ctx int thread_id; }; +static inline int ctx_actually_replaced(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + ctx->param->action == MA_ACTION_REPLACE && + ctx->rep_ctx->actually_replaced==1) + { + return 1; + } + else + { + return 0; + } +} + +static inline int ctx_actually_ran_script(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + ctx->param->action == MA_ACTION_LUA_SCRIPT && + ctx->tsg_ctx->actually_executed==1) + { + return 1; + } + else + { + return 0; + } +} + +static inline int ctx_actually_inserted(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + ctx->param->action == MA_ACTION_INSERT && + ctx->ins_ctx->actually_inserted==1) + { + return 1; + } + else + { + return 0; + } +} + +static inline int ctx_actually_edited(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + ctx->param->action == MA_ACTION_ELEMENT && ctx->edit_ctx != NULL && + ctx->edit_ctx->actually_edited==1) + { + return 1; + } + else + { + return 0; + } +} + +static inline int ctx_actually_manipulate(struct proxy_http_ctx * ctx) +{ + if(ctx->action == PX_ACTION_MANIPULATE && + (ctx->param->action == MA_ACTION_REDIRECT || + ctx->param->action == MA_ACTION_HIJACK)&& + ctx->manipulate_replaced==1) + { + return 1; + } + else + { + return 0; + } +} + void http_repl_ctx_free(struct replace_ctx* rep_ctx) { if (rep_ctx->http_body) @@ -2743,6 +2814,8 @@ enum proxy_action http_scan(const struct tfe_http_session * session, enum tfe_ht if ((events & EV_HTTP_REQ_BODY_END) | (events & EV_HTTP_RESP_BODY_END)) { + table_id = events & EV_HTTP_REQ_BODY_END ? g_proxy_rt->scan_table_id[PXY_CTRL_HTTP_REQ_BODY] : g_proxy_rt + ->scan_table_id[PXY_CTRL_HTTP_RES_BODY]; scan_ret = maat_scan_not_logic(g_proxy_rt->feather, table_id, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &n_hit_result, ctx->scan_mid); if (scan_ret == MAAT_SCAN_HIT) @@ -2829,7 +2902,12 @@ void enforce_control_policy(const struct tfe_stream * stream, const struct tfe_h if(ctx->log_resp_body == NULL) ctx->log_resp_body = evbuffer_new(); evbuffer_add(ctx->log_resp_body, body_frag, frag_size); } - proxy_send_metric_log(stream, ctx, thread_id, 1); + + if((((ctx_actually_replaced(ctx)) || (ctx_actually_inserted(ctx)) || (ctx_actually_edited(ctx)) || (ctx_actually_manipulate(ctx)) + || ctx_actually_ran_script(ctx)) || ctx->action == PX_ACTION_REJECT || (ctx->action == PX_ACTION_MONIT))) + { + proxy_send_metric_log(stream, ctx, thread_id, 1); + } return; } @@ -2910,80 +2988,6 @@ void proxy_on_http_begin(const struct tfe_stream *stream, const struct tfe_http_ return; } -static inline int ctx_actually_replaced(struct proxy_http_ctx * ctx) -{ - - if(ctx->action == PX_ACTION_MANIPULATE && - ctx->param->action == MA_ACTION_REPLACE && - ctx->rep_ctx->actually_replaced==1) - { - return 1; - } - else - { - return 0; - } -} - -static inline int ctx_actually_ran_script(struct proxy_http_ctx * ctx) -{ - if(ctx->action == PX_ACTION_MANIPULATE && - ctx->param->action == MA_ACTION_LUA_SCRIPT && - ctx->tsg_ctx->actually_executed==1) - { - return 1; - } - else - { - return 0; - } -} - -static inline int ctx_actually_inserted(struct proxy_http_ctx * ctx) -{ - - if(ctx->action == PX_ACTION_MANIPULATE && - ctx->param->action == MA_ACTION_INSERT && - ctx->ins_ctx->actually_inserted==1) - { - return 1; - } - else - { - return 0; - } -} - -static inline int ctx_actually_edited(struct proxy_http_ctx * ctx) -{ - - if(ctx->action == PX_ACTION_MANIPULATE && - ctx->param->action == MA_ACTION_ELEMENT && ctx->edit_ctx != NULL && - ctx->edit_ctx->actually_edited==1) - { - return 1; - } - else - { - return 0; - } -} - -static inline int ctx_actually_manipulate(struct proxy_http_ctx * ctx) -{ - if(ctx->action == PX_ACTION_MANIPULATE && - (ctx->param->action == MA_ACTION_REDIRECT || - ctx->param->action == MA_ACTION_HIJACK)&& - ctx->manipulate_replaced==1) - { - return 1; - } - else - { - return 0; - } -} - void proxy_on_http_end(const struct tfe_stream * stream, const struct tfe_http_session * session, unsigned int thread_id, void ** pme) { diff --git a/plugin/business/tsg-http/src/tsg_logger.cpp b/plugin/business/tsg-http/src/tsg_logger.cpp index 6673d83..4ae7203 100644 --- a/plugin/business/tsg-http/src/tsg_logger.cpp +++ b/plugin/business/tsg-http/src/tsg_logger.cpp @@ -253,7 +253,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]); cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP"); - cJSON_AddStringToObject(common_obj, "ip_protocol", "TCP"); + cJSON_AddStringToObject(common_obj, "ip_protocol", "tcp"); cJSON_AddNumberToObject(common_obj, "out_link_id", 0); cJSON_AddNumberToObject(common_obj, "in_link_id", 0); cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str); diff --git a/resource/pangu/table_info.conf b/resource/pangu/table_info.conf index e16534f..6b27e75 100644 --- a/resource/pangu/table_info.conf +++ b/resource/pangu/table_info.conf @@ -66,15 +66,16 @@ "table_id":5, "table_name":"TSG_OBJ_IP", "db_tables":["TSG_OBJ_IP_ADDR","TSG_OBJ_IP_LEARNING_ADDR"], - "table_type":"ip_plus", - "valid_column":7, + "table_type":"ip", + "valid_column":8, "custom": { "item_id":1, "group_id":2, "addr_type":3, "addr_format":4, "ip1":5, - "ip2":6 + "ip2":6, + "port":7 } }, {