From fbbbfadb0c4dd29d70685dfbfd9f736087782d3f Mon Sep 17 00:00:00 2001 From: zhengchao Date: Mon, 19 Nov 2018 18:37:11 +0800 Subject: [PATCH 1/5] =?UTF-8?q?1.=20=E4=BF=AE=E5=A4=8D=E5=91=BD=E4=B8=AD?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E7=AD=96=E7=95=A5=E5=90=8E=EF=BC=8C=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E4=B8=8A=E4=BC=A0=E7=9A=84=E5=A4=84=E7=90=86=E9=80=BB?= =?UTF-8?q?=E8=BE=91=202.=20=E6=94=AF=E6=8C=81=E6=96=B0=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E7=9A=84FieldStat=E7=9A=84Histogram=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/src/tfe_future.cpp | 4 +-- plugin/business/pangu-http/src/pangu_http.cpp | 13 +++++---- .../pangu-http/src/pangu_web_cache.cpp | 27 ++++++++++++++----- .../business/pangu-http/src/pangu_web_cache.h | 11 ++------ 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/common/src/tfe_future.cpp b/common/src/tfe_future.cpp index 3015aa8..a39ebeb 100644 --- a/common/src/tfe_future.cpp +++ b/common/src/tfe_future.cpp @@ -10,7 +10,7 @@ #include -const char* FP_HISTOGRAM_BINS="10,50,100,500"; +const char* FP_HISTOGRAM_BINS="0.5,0.8,0.9,0.95"; struct future_promise_instance { @@ -110,7 +110,7 @@ static long field_get_set_cb(void * data, const uchar * key, uint size, void * u { field_id=(int*)malloc(sizeof(int)*2); snprintf(buff,sizeof(buff),"%s(ms)",(char*)key); - field_id[0]=FS_register(args->fs_handle, FS_STYLE_HISTOGRAM, FS_CALC_CURRENT, buff); + field_id[0]=FS_register_histogram(args->fs_handle, FS_CALC_SPEED, buff, 1, 5*1000, 2);//1ms~5s args->fsid_latency=field_id[0]; snprintf(buff,sizeof(buff),"%s%s",(char*)key,fail_str); field_id[1]=FS_register(args->fs_handle, FS_STYLE_FIELD, FS_CALC_SPEED,buff); diff --git a/plugin/business/pangu-http/src/pangu_http.cpp b/plugin/business/pangu-http/src/pangu_http.cpp index f2aa2b4..e55e563 100644 --- a/plugin/business/pangu-http/src/pangu_http.cpp +++ b/plugin/business/pangu-http/src/pangu_http.cpp @@ -441,7 +441,6 @@ struct pangu_http_ctx size_t datalen, unsigned int thread_id, struct pangu_http_ctx* ctx); enum cache_pending_result pending_result; - enum cache_query_status cache_query_status; struct future *f_cache_pending, *f_cache_query; struct tfe_http_session * ref_session; struct tfe_http_half* cache_revalidate_req; @@ -1086,16 +1085,16 @@ static void cache_query_on_succ(future_result_t * result, void * user) break; case CACHE_QUERY_RESULT_END: assert(ctx->cached_response!=NULL); - ctx->cache_query_status=WEB_CACHE_HIT; tfe_http_half_write_body_end(ctx->cached_response); - //ownership has been transferred to http session, set to NULL. + //ownership has been transferred to http session, set to NULL. + ctx->pending_result=PENDING_RESULT_HIT; ctx->cached_response=NULL; assert(ctx->cache_result_actual_sz==ctx->cache_result_declared_sz); future_destroy(ctx->f_cache_query); ctx->f_cache_query=NULL; break; case CACHE_QUERY_RESULT_MISS: - ctx->cache_query_status=WEB_CACHE_MISS; + ctx->pending_result=PENDING_RESULT_MISS; ctx->resumed_cb=dummy_resume; tfe_http_session_resume(ctx->ref_session); future_destroy(ctx->f_cache_query); @@ -1111,7 +1110,6 @@ static void cache_query_on_fail(enum e_future_error err, const char * what, void struct pangu_http_ctx * ctx = (struct pangu_http_ctx *)user; future_destroy(ctx->f_cache_query); ctx->f_cache_query=NULL; - ctx->cache_query_status=WEB_CACHE_MISS; if(!ctx->cached_response) { tfe_http_session_resume(ctx->ref_session); @@ -1122,7 +1120,7 @@ static void cache_query_on_fail(enum e_future_error err, const char * what, void tfe_http_half_write_body_end(ctx->cached_response); ctx->cached_response=NULL; } - + ctx->pending_result=PENDING_RESULT_MISS; printf("cache query failed: %s %s\n", ctx->ref_session->req->req_spec.url, what); } static void cache_pending_on_succ(future_result_t * result, void * user) @@ -1347,7 +1345,8 @@ void pangu_on_http_data(const struct tfe_stream * stream, const struct tfe_http_ } } - if(tfe_http_in_response(events)&& ctx->pending_result==PENDING_RESULT_MISS) + if(tfe_http_in_response(events) + && (ctx->pending_result==PENDING_RESULT_MISS||ctx->pending_result==PENDING_RESULT_ALLOWED)) { cache_update(session, events, body_frag, frag_size, thread_id, ctx); } diff --git a/plugin/business/pangu-http/src/pangu_web_cache.cpp b/plugin/business/pangu-http/src/pangu_web_cache.cpp index ac39eef..c032cf6 100644 --- a/plugin/business/pangu-http/src/pangu_web_cache.cpp +++ b/plugin/business/pangu-http/src/pangu_web_cache.cpp @@ -190,7 +190,7 @@ void cache_stat_init(struct cache_handle* cache) { const char* fieldstat_output="./cache.fieldstat"; const char* app_name="tango_cache"; - const char* obj_size_bins_KB="10,100,1000,10000"; + const char* obj_size_bins_KB="0.5,0.8,0.9,0.95"; int value=0, i=0; screen_stat_handle_t fs_handle=NULL; @@ -238,7 +238,14 @@ void cache_stat_init(struct cache_handle* cache) for(i=0;i<__CACHE_STAT_MAX;i++) { - cache->fs_id[i]=FS_register(cache->fs_handle, spec[i].style, spec[i].calc_type, spec[i].name); + if(spec[i].style==FS_STYLE_HISTOGRAM) + { + cache->fs_id[i]=FS_register_histogram(cache->fs_handle, spec[i].calc_type, spec[i].name,1,10*1024*1024,2); + } + else + { + cache->fs_id[i]=FS_register(cache->fs_handle, spec[i].style, spec[i].calc_type, spec[i].name); + } } // value=cache->fs_id[STAT_CACHE_QUERY_HIT]; // FS_set_para(cache->fs_handle, ID_INVISBLE, &value, sizeof(value)); @@ -492,7 +499,6 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de key_desc=cJSON_GetObjectItem(json,"cache_key"); if(key_desc && key_desc->type==cJSON_Object) { - param->key_descr.is_not_empty=1; qs=cJSON_GetObjectItem(key_desc,"ignore_qs"); if(qs && qs->type==cJSON_Array) { @@ -508,8 +514,15 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de } } item=cJSON_GetObjectItem(key_desc,"cookie"); - if(item && item->type==cJSON_String) param->key_descr.include_cookie=tfe_strdup(item->valuestring); - + if(item && item->type==cJSON_String) + { + param->key_descr.include_cookie=tfe_strdup(item->valuestring); + + } + if(param->key_descr.qs_num>0||param->key_descr.include_cookie!=NULL) + { + param->key_descr.is_not_empty=1; + } } @@ -1005,7 +1018,7 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u { _mid->cache_key=get_cache_key(request, &(param->key_descr)); } - TFE_LOG_DEBUG(handle->logger, "cache policy %d matched: url=%s alternative key=%s", + TFE_LOG_DEBUG(handle->logger, "cache policy %d matched: url=%s alt-key=%s", cache_policy.config_id, request->req_spec.url, _mid->cache_key!=NULL?_mid->cache_key:"null"); @@ -1204,7 +1217,7 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle, case ALLOWED: case UNDEFINED: if(_mid->shall_bypass - || content_len > param->max_cache_obj_size + || (param->max_cache_obj_size!=0 && content_len > param->max_cache_obj_size) || (!param->cache_cookied_cont && _mid->has_cookie) || (!param->cache_html && _mid->is_html)) { diff --git a/plugin/business/pangu-http/src/pangu_web_cache.h b/plugin/business/pangu-http/src/pangu_web_cache.h index dcd6bb9..18cfa65 100644 --- a/plugin/business/pangu-http/src/pangu_web_cache.h +++ b/plugin/business/pangu-http/src/pangu_web_cache.h @@ -4,15 +4,7 @@ #include #include -enum cache_query_status -{ - WEB_CACHE_BEFORE_QUERY=0, - WEB_CACHE_NOT_APPLICABLE, - WEB_CACHE_NEED_VERIFY, - WEB_CACHE_QUERY_DATA, - WEB_CACHE_MISS, - WEB_CACHE_HIT -}; + struct cache_handle; struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, struct event_base* gc_evbase, Maat_feather_t feather, void *logger); @@ -49,6 +41,7 @@ enum cache_pending_result PENDING_RESULT_FOBIDDEN, PENDING_RESULT_REVALIDATE, PENDING_RESULT_ALLOWED, + PENDING_RESULT_HIT, PENDING_RESULT_MISS }; struct cache_mid; From b47566cf4b4cdd7ca91de0298b9e09cc83446766 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 20 Nov 2018 15:49:25 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dtfe=5Fhttp=5Fin=5Frequest?= =?UTF-8?q?=E4=B8=AD=E6=BC=8F=E5=88=A4EV=5FHTTP=5FREQ=5FEND=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E7=9A=84bug=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/include/tfe_http.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/include/tfe_http.h b/common/include/tfe_http.h index 0384e13..327d84d 100644 --- a/common/include/tfe_http.h +++ b/common/include/tfe_http.h @@ -430,7 +430,7 @@ static inline struct tfe_http_half * tfe_http_session_response_create(struct tfe static inline int tfe_http_in_request(enum tfe_http_event events) { if ((events & EV_HTTP_REQ_HDR) | (events & EV_HTTP_REQ_BODY_BEGIN) | (events & EV_HTTP_REQ_BODY_END) - | (events & EV_HTTP_REQ_BODY_CONT)) + | (events & EV_HTTP_REQ_BODY_CONT)| (events & EV_HTTP_REQ_END)) { return 1; } From 397d966a0f88555ddf95f30cae1939d05a9f6ac3 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 20 Nov 2018 15:56:13 +0800 Subject: [PATCH 3/5] =?UTF-8?q?#86=20=E4=BF=AE=E6=AD=A3=E5=9C=A8HTTP?= =?UTF-8?q?=E8=AF=B7=E6=B1=82=E5=9C=A8Resume()=E6=93=8D=E4=BD=9C=E5=90=8E?= =?UTF-8?q?=E8=AE=BE=E7=BD=AEHTTP=E5=BA=94=E7=AD=94=EF=BC=8C=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E4=B8=A2=E5=BC=83=E5=8E=9F=E5=A7=8BHTTP=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E7=9A=84=E9=97=AE=E9=A2=98=E3=80=82=20*=20=E5=8E=9F?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E5=9C=A8=E7=94=A8=E6=88=B7=E8=B0=83=E7=94=A8?= =?UTF-8?q?resume()=E5=90=8E=EF=BC=8C=E6=B2=A1=E6=9C=89=E5=B0=86=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E8=AE=BE=E7=BD=AE=E7=9A=84user=5Fstream=5Faction?= =?UTF-8?q?=E4=BC=A0=E9=80=92=E5=88=B0=E5=8E=9F=E5=A7=8BHTTP=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=EF=BC=8C=E5=AF=BC=E8=87=B4=E8=AF=A5=E5=8E=9F=E5=A7=8B?= =?UTF-8?q?=E8=AF=B7=E6=B1=82=E6=B2=A1=E6=9C=89=E8=A2=AB=E4=B8=A2=E5=BC=83?= =?UTF-8?q?=EF=BC=8C=E7=8E=B0=E4=BF=AE=E6=AD=A3=E3=80=82@luqiuwen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/protocol/http/src/http_entry.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugin/protocol/http/src/http_entry.cpp b/plugin/protocol/http/src/http_entry.cpp index 44348ba..e29e6b4 100644 --- a/plugin/protocol/http/src/http_entry.cpp +++ b/plugin/protocol/http/src/http_entry.cpp @@ -392,6 +392,12 @@ enum tfe_stream_action http_connection_entry(const struct tfe_stream * stream, e hs_private->release_lock--; hs_private->suspend_counter++; + /* Retrieve the user's stream action, because the user may set request/response at resume's callback */ + if(hf_private_in->is_user_stream_action_set) + { + hf_private_in->stream_action = hf_private_in->user_stream_action; + } + /* Call user callback, tell user we resume from suspend */ assert(hs_private->resume_tag_singal); hs_private->resume_tag_singal = false; From dccac95d1614946cc468dbbdbd266345652587c2 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 20 Nov 2018 16:42:35 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E4=BF=AE=E6=AD=A3stream=5Fwrite=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E8=BF=9E=E6=8E=A5=E5=85=B3=E9=97=AD=E5=A4=84=E7=90=86?= =?UTF-8?q?=E4=B8=AD=E6=B2=A1=E6=9C=89=E9=80=9A=E7=9F=A5write=5Fctx?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98=E3=80=82=20*=20=E5=8E=9F=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E5=A4=84=E7=90=86stream=5Fwrite=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E7=9A=84=E5=85=B3=E9=97=AD=E4=BA=8B=E4=BB=B6=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E5=B0=86write=5Fctx=E4=B8=AD=E7=9A=84?= =?UTF-8?q?=E5=BC=95=E7=94=A8=E6=8C=87=E9=92=88=E7=BD=AE=E4=BD=8D=E3=80=82?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E4=B8=8A=E5=B1=82=E7=BB=A7=E7=BB=ADfrag=5Fwr?= =?UTF-8?q?ite=E6=97=B6=EF=BC=8C=E6=B2=A1=E6=9C=89=E6=9C=89=E6=95=88?= =?UTF-8?q?=E7=9A=84=E5=BA=95=E5=B1=82=E8=BF=9E=E6=8E=A5=E3=80=82=E7=8E=B0?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- platform/src/tcp_stream.cpp | 43 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/platform/src/tcp_stream.cpp b/platform/src/tcp_stream.cpp index 4c3f52f..3e9e170 100644 --- a/platform/src/tcp_stream.cpp +++ b/platform/src/tcp_stream.cpp @@ -208,6 +208,11 @@ struct tfe_stream_write_ctx * tfe_stream_write_frag_start(const struct tfe_strea int tfe_stream_write_frag(struct tfe_stream_write_ctx * w_ctx, const unsigned char * data, size_t size) { + if(w_ctx->_stream == NULL) + { + return -EPIPE; + } + struct tfe_conn_private * this_conn = __this_conn(w_ctx->_stream, w_ctx->dir); int ret = 0; if (this_conn != NULL) @@ -224,8 +229,13 @@ int tfe_stream_write_frag(struct tfe_stream_write_ctx * w_ctx, const unsigned ch void tfe_stream_write_frag_end(struct tfe_stream_write_ctx * w_ctx) { - struct tfe_conn_private * this_conn = __this_conn(w_ctx->_stream, w_ctx->dir); - struct tfe_conn_private * peer_conn = __peer_conn(w_ctx->_stream, w_ctx->dir); + struct tfe_conn_private * this_conn; + struct tfe_conn_private * peer_conn; + + /* The connection terminated before this function call */ + if (w_ctx->_stream == NULL) goto __out; + this_conn = __this_conn(w_ctx->_stream, w_ctx->dir); + peer_conn = __peer_conn(w_ctx->_stream, w_ctx->dir); if (this_conn != NULL) { @@ -233,20 +243,18 @@ void tfe_stream_write_frag_end(struct tfe_stream_write_ctx * w_ctx) bufferevent_enable(peer_conn->bev, EV_READ); } - if (w_ctx->_stream != NULL) + if (w_ctx->dir == CONN_DIR_DOWNSTREAM) { - if (w_ctx->dir == CONN_DIR_DOWNSTREAM) - { - assert(w_ctx->_stream->w_ctx_downstream == w_ctx); - w_ctx->_stream->w_ctx_downstream = NULL; - } - else - { - assert(w_ctx->_stream->w_ctx_upstream == w_ctx); - w_ctx->_stream->w_ctx_upstream = NULL; - } + assert(w_ctx->_stream->w_ctx_downstream == w_ctx); + w_ctx->_stream->w_ctx_downstream = NULL; + } + else + { + assert(w_ctx->_stream->w_ctx_upstream == w_ctx); + w_ctx->_stream->w_ctx_upstream = NULL; } +__out: free(w_ctx); } @@ -616,6 +624,7 @@ static void __stream_bev_eventcb(struct bufferevent * bev, short events, void * struct tfe_conn_private ** ref_peer_conn{}; struct ssl_stream ** ref_this_ssl_stream{}; struct ssl_stream ** ref_peer_ssl_stream{}; + struct tfe_stream_write_ctx ** ref_this_write_ctx{}; const char * __str_dir = NULL; if (__bev_dir(_stream, bev) == CONN_DIR_UPSTREAM) @@ -624,6 +633,7 @@ static void __stream_bev_eventcb(struct bufferevent * bev, short events, void * ref_peer_conn = &_stream->conn_downstream; ref_this_ssl_stream = &_stream->ssl_upstream; ref_peer_ssl_stream = &_stream->ssl_downstream; + ref_this_write_ctx = &_stream->w_ctx_upstream; } if (__bev_dir(_stream, bev) == CONN_DIR_DOWNSTREAM) @@ -632,6 +642,7 @@ static void __stream_bev_eventcb(struct bufferevent * bev, short events, void * ref_peer_conn = &_stream->conn_upstream; ref_this_ssl_stream = &_stream->ssl_downstream; ref_peer_ssl_stream = &_stream->ssl_upstream; + ref_this_write_ctx = &_stream->w_ctx_downstream; } if (events & BEV_EVENT_ERROR || events & BEV_EVENT_EOF) @@ -665,6 +676,12 @@ __close_connection: if (*ref_this_conn != NULL) { + /* There is a frag writter setted, need to clear the reference of stream in the writter to indicate the connection is closed */ + if(*ref_this_write_ctx != NULL) + { + (*ref_this_write_ctx)->_stream = NULL; + } + __conn_private_destory_with_ssl(ev_base, *ref_this_conn, *ref_this_ssl_stream); *ref_this_conn = NULL; *ref_this_ssl_stream = NULL; From 47e45870ac5f85cfcbb8a184cf48ea0a1543a3b1 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 20 Nov 2018 16:43:17 +0800 Subject: [PATCH 5/5] =?UTF-8?q?miss=E7=8A=B6=E6=80=81=E6=89=8D=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E7=BC=93=E5=AD=98=E6=9B=B4=E6=96=B0=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/business/pangu-http/src/pangu_http.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugin/business/pangu-http/src/pangu_http.cpp b/plugin/business/pangu-http/src/pangu_http.cpp index e55e563..d44fac8 100644 --- a/plugin/business/pangu-http/src/pangu_http.cpp +++ b/plugin/business/pangu-http/src/pangu_http.cpp @@ -1345,8 +1345,7 @@ void pangu_on_http_data(const struct tfe_stream * stream, const struct tfe_http_ } } - if(tfe_http_in_response(events) - && (ctx->pending_result==PENDING_RESULT_MISS||ctx->pending_result==PENDING_RESULT_ALLOWED)) + if(tfe_http_in_response(events) && ctx->pending_result==PENDING_RESULT_MISS) { cache_update(session, events, body_frag, frag_size, thread_id, ctx); }