From dc6a6331d31773f08a363f3d9d49236bb02dc905 Mon Sep 17 00:00:00 2001 From: zhangchengwei Date: Sun, 23 Sep 2018 18:13:46 +0800 Subject: [PATCH] =?UTF-8?q?GET=E6=97=B6HEADER=E5=8F=AA=E5=9B=9E=E8=B0=83?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=EF=BC=9B=E5=A2=9E=E5=8A=A0CACHE=20MISS?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cache/Makefile | 2 +- cache/include/tango_cache_client.h | 11 +-- cache/tango_cache_client.cpp | 20 ++++-- cache/tango_cache_transfer.cpp | 99 +++++++++++---------------- cache/test_demo/cache_evbase_test.cpp | 4 +- cache/test_demo/pangu_tg_cahce.conf | 2 +- cache/test_demo/tango_cache_test.c | 4 +- 7 files changed, 65 insertions(+), 77 deletions(-) diff --git a/cache/Makefile b/cache/Makefile index 41b3a25..0a93a29 100644 --- a/cache/Makefile +++ b/cache/Makefile @@ -6,7 +6,7 @@ LDFLAGS = LIBS = -lMESA_handle_logger -lMESA_htable -lMESA_prof_load -lwiredcfg LIBS += -lssl -lcrypto -lcurl -levent -lxml2 -OBJS = tfe_future.o tango_cache_tools.o tango_cache_client.o tango_cache_transfer.o tango_cache_xml.o cache_evbase_client.o +OBJS = tango_cache_tools.o tango_cache_client.o tango_cache_transfer.o tango_cache_xml.o cache_evbase_client.o TARGET_A=pangu_tango_cache.a diff --git a/cache/include/tango_cache_client.h b/cache/include/tango_cache_client.h index 9e5c0cb..03e0d52 100644 --- a/cache/include/tango_cache_client.h +++ b/cache/include/tango_cache_client.h @@ -34,18 +34,19 @@ struct cache_statistics { long long get_recv_num; //发起GET的次数 long long get_succ_num; //GET成功的次数 - long long get_fail_num; //GET失败的次数 + long long get_miss_num; //GET未命中的次数 + long long get_error_num;//GET失败的次数 long long put_recv_num; //发起UPLOAD的次数 long long put_succ_num; //UPLOAD成功的次数 - long long put_fail_num; //UPLOAD失败的次数 + long long put_error_num;//UPLOAD失败的次数 long long memory_used; //当前UPLOAD BODY所占内存大小 long long session_num; //当前正在进行GET/PUT的HTTP会话数 }; enum CACHE_RESULT_TYPE { - RESULT_TYPE_HEADER=0, - RESULT_TYPE_USERTAG, + RESULT_TYPE_HEADER=0, //只调用一次 + RESULT_TYPE_USERTAG, //只调用一次 RESULT_TYPE_BODY, }; @@ -74,7 +75,7 @@ struct tango_cache_meta const char* usertag; //可以是任意内容，GET时会原样返回 size_t usertag_len; //最大长度USER_TAG_MAX_LEN，0表示没有该头部 union{ - struct response_freshness put;//TODO + struct response_freshness put; struct request_freshness get; }; }; diff --git a/cache/tango_cache_client.cpp b/cache/tango_cache_client.cpp index 2735ea2..f413b4b 100644 --- a/cache/tango_cache_client.cpp +++ b/cache/tango_cache_client.cpp @@ -46,10 +46,11 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str { out->get_recv_num = instance->statistic.get_recv_num; out->get_succ_num = instance->statistic.get_succ_num; - out->get_fail_num = instance->statistic.get_fail_num; + out->get_error_num= instance->statistic.get_error_num; + out->get_miss_num = instance->statistic.get_miss_num; out->put_recv_num = instance->statistic.put_recv_num; out->put_succ_num = instance->statistic.put_succ_num; - out->put_fail_num = instance->statistic.put_fail_num; + out->put_error_num= instance->statistic.put_error_num; out->session_num = instance->statistic.session_num; out->memory_used = instance->statistic.memory_used; } @@ -64,13 +65,13 @@ const char *tango_cache_get_object_key(struct tango_cache_ctx *ctx) return ctx->object_key; } -static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, struct cache_statistics *statistic) +static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, enum CACHE_ERR_CODE error_code, struct cache_statistics *statistic) { if(method == CACHE_REQUEST_PUT) { if(fail_state) { - statistic->put_fail_num += 1; + statistic->put_error_num += 1; } else { @@ -81,7 +82,10 @@ static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail { if(fail_state) { - statistic->get_fail_num += 1; + if(error_code == CACHE_ERR_CURL) + statistic->get_error_num += 1; + else + statistic->get_miss_num += 1; } else { @@ -154,7 +158,7 @@ void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx) } } } - update_statistics(ctx->method, ctx->fail_state, &ctx->instance->statistic); + update_statistics(ctx->method, ctx->fail_state, ctx->error_code, &ctx->instance->statistic); free(ctx); } @@ -265,6 +269,10 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance * ctx->headers = curl_slist_append(ctx->headers, meta->std_hdr[i]); } } + if(meta->std_hdr[HDR_CONTENT_TYPE] == NULL) + { + ctx->headers = curl_slist_append(ctx->headers, "Content-Type:"); + } //其他定义的头部，GET时会原样返回 if(meta->usertag_len>0 && meta->usertag_len<=USER_TAG_MAX_LEN) { diff --git a/cache/tango_cache_transfer.cpp b/cache/tango_cache_transfer.cpp index 31895b5..251055b 100644 --- a/cache/tango_cache_transfer.cpp +++ b/cache/tango_cache_transfer.cpp @@ -158,7 +158,7 @@ static size_t curl_write_uploadID_cb(void *ptr, size_t size, size_t count, void if(code != CURLE_OK || ctx->res_code!=200L) { ctx->fail_state = true; - ctx->error_code = CACHE_CACHE_MISS; + ctx->error_code = CACHE_ERR_CURL; if(code != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error); return size*count; } @@ -339,6 +339,7 @@ int http_put_complete_part_evbuf(struct tango_cache_ctx *ctx) if(ret <= 0) { ctx->fail_state = true; + ctx->error_code = CACHE_ERR_CURL; tango_cache_ctx_destroy(ctx); } } @@ -380,6 +381,7 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx) else { ctx->fail_state = true; + ctx->error_code = CACHE_ERR_CURL; tango_cache_ctx_destroy(ctx); } } @@ -389,6 +391,7 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx) if(ret <= 0) { ctx->fail_state = true; + ctx->error_code = CACHE_ERR_CURL; if(cache_cancel_upload_minio(ctx)) { ctx->put_state = PUT_STATE_CANCEL; @@ -421,6 +424,10 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode ctx->error_code = CACHE_ERR_CURL; ctx->fail_state = true; if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error); + if(ctx->close_state) + { + tango_cache_ctx_destroy(ctx); + } } else { @@ -445,6 +452,7 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode if(res != CURLE_OK || res_code!=200L) { ctx->fail_state = true; + ctx->error_code = CACHE_ERR_CURL; if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error); } if(ctx->fail_state) @@ -483,6 +491,7 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode if(res != CURLE_OK || res_code!=200L) { ctx->fail_state = true; + ctx->error_code = CACHE_ERR_CURL; if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error); } tango_cache_ctx_destroy(ctx); @@ -577,7 +586,7 @@ void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode { if(res!=CURLE_OK || res_code!=200L) { - ctx->error_code = CACHE_ERR_CURL; + ctx->error_code = (res!=CURLE_OK)?CACHE_ERR_CURL:CACHE_CACHE_MISS; promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, ctx->error); } else @@ -619,17 +628,32 @@ static size_t curl_get_response_body_cb(void *ptr, size_t size, size_t count, vo if(ctx->need_hdrs!=RESPONSE_HDR_ALL) //无Expires时 { ctx->fail_state = true; - ctx->error_code = CACHE_CACHE_MISS; + ctx->error_code = CACHE_ERR_CURL; ctx->get_state = GET_STATE_DELETE; promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache Expires or last-modify not found"); + return size*count; } - else + + if(ctx->response_tag.len > 0) { - result.data_frag = ptr; - result.size = size * count; - result.type = RESULT_TYPE_BODY; + result.data_frag = ctx->response_tag.buff; + result.size = ctx->response_tag.len; + result.type = RESULT_TYPE_USERTAG; promise_success(future_to_promise(ctx->future), &result); + easy_string_destroy(&ctx->response_tag); } + if(ctx->response.len > 0) + { + result.data_frag = ctx->response.buff; + result.size = ctx->response.len; + result.type = RESULT_TYPE_HEADER; + promise_success(future_to_promise(ctx->future), &result); + easy_string_destroy(&ctx->response); + } + result.data_frag = ptr; + result.size = size * count; + result.type = RESULT_TYPE_BODY; + promise_success(future_to_promise(ctx->future), &result); return size*count; } @@ -654,7 +678,6 @@ static bool check_expires_header(struct tango_cache_ctx *ctx, const char *expire static bool check_fresh_header(struct tango_cache_ctx *ctx) { - struct tango_cache_result result; time_t now_gmt; if(ctx->need_hdrs != RESPONSE_HDR_ALL) @@ -669,23 +692,6 @@ static bool check_fresh_header(struct tango_cache_ctx *ctx) promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not fresh"); return false; } - - if(ctx->response_tag.len > 0) - { - result.data_frag = ctx->response_tag.buff; - result.size = ctx->response_tag.len; - result.type = RESULT_TYPE_USERTAG; - promise_success(future_to_promise(ctx->future), &result); - easy_string_destroy(&ctx->response_tag); - } - if(ctx->response.len > 0) - { - result.data_frag = ctx->response.buff; - result.size = ctx->response.len; - result.type = RESULT_TYPE_HEADER; - promise_success(future_to_promise(ctx->future), &result); - easy_string_destroy(&ctx->response); - } return true; } @@ -697,7 +703,7 @@ static bool check_get_result_code(struct tango_cache_ctx *ctx) if(code != CURLE_OK || ctx->res_code!=200L) { ctx->fail_state = true; - ctx->error_code = CACHE_CACHE_MISS; + ctx->error_code = (code!=CURLE_OK)?CACHE_ERR_CURL:CACHE_CACHE_MISS; promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, (code!=CURLE_OK)?ctx->error:"cache not hit"); return false; } @@ -707,9 +713,7 @@ static bool check_get_result_code(struct tango_cache_ctx *ctx) static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, void *userp) { struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; - struct tango_cache_result result; - char *start=(char *)ptr, *pos_colon, *hdrdata=(char*)ptr; - bool ptr_valid=false; + char *start=(char *)ptr, *pos_colon; size_t raw_len = size*count, hdrlen=size*count; char usertag[2048]; @@ -722,8 +726,6 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, return raw_len; } - result.type = RESULT_TYPE_HEADER; - if((pos_colon=(char*)memchr(start, ':', raw_len))!=NULL) { size_t datalen = pos_colon - start; @@ -755,41 +757,18 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, { if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0) { - if(ctx->need_hdrs == RESPONSE_HDR_ALL) - { - ptr_valid = true; - result.type = RESULT_TYPE_USERTAG; - hdrdata = usertag; - } - else - { - easy_string_savedata(&ctx->response_tag, usertag, hdrlen); - } + easy_string_savedata(&ctx->response_tag, usertag, hdrlen); } } break; - case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) ptr_valid = true; break; - case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) ptr_valid = true; break; - case 14: if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) ptr_valid = true; break; - case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) ptr_valid = true; break; - case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) ptr_valid = true; break; + case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; + case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; + case 14: if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; + case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; + case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; default: break; } } - - if(ptr_valid) - { - if(ctx->need_hdrs == RESPONSE_HDR_ALL) - { - result.data_frag = hdrdata; - result.size = hdrlen; - promise_success(future_to_promise(ctx->future), &result); - } - else if(result.type == RESULT_TYPE_HEADER) - { - easy_string_savedata(&ctx->response, hdrdata, hdrlen); - } - } return raw_len; } diff --git a/cache/test_demo/cache_evbase_test.cpp b/cache/test_demo/cache_evbase_test.cpp index 95b7c40..1b6f572 100644 --- a/cache/test_demo/cache_evbase_test.cpp +++ b/cache/test_demo/cache_evbase_test.cpp @@ -248,8 +248,8 @@ int main(int argc, char **argv) struct cache_statistics out; cache_evbase_get_statistics(instance_asyn, &out); - printf("get_recv: %llu, get_succ: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n", - out.get_recv_num, out.get_succ_num, out.get_fail_num, out.put_recv_num, out.put_succ_num, out.put_fail_num, out.session_num, out.memory_used); + printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n", + out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.session_num, out.memory_used); return 0; } diff --git a/cache/test_demo/pangu_tg_cahce.conf b/cache/test_demo/pangu_tg_cahce.conf index c398565..fa7e465 100644 --- a/cache/test_demo/pangu_tg_cahce.conf +++ b/cache/test_demo/pangu_tg_cahce.conf @@ -15,7 +15,7 @@ CACHE_BLOCK_MAX_SIZE=8192 MAX_USED_MEMORY_SIZE_MB=5120 #涓婁紶鏃禘xpires澶撮儴鐨勮繃鏈熸椂闂达紝鍗曚綅绉掞紝鏈灏60锛1鍒嗛挓锛 -CACHE_DEFAULT_TTL_SECOND=60 +CACHE_DEFAULT_TTL_SECOND=3600 #鏄惁瀵瑰璞＄殑鍚嶇О杩涜鍝堝笇锛屽紑鍚搱甯屾湁鍔╀簬鎻愰珮涓婁紶涓嬭浇鐨勯熺巼 CACHE_OBJECT_KEY_HASH_SWITCH=1 diff --git a/cache/test_demo/tango_cache_test.c b/cache/test_demo/tango_cache_test.c index 4ed976c..4cb6394 100644 --- a/cache/test_demo/tango_cache_test.c +++ b/cache/test_demo/tango_cache_test.c @@ -284,8 +284,8 @@ void timer_cb(evutil_socket_t fd, short what, void *arg) }*/ tango_cache_get_statistics(tango_instance, &out); - printf("get_recv: %llu, get_succ: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n", - out.get_recv_num, out.get_succ_num, out.get_fail_num, out.put_recv_num, out.put_succ_num, out.put_fail_num, out.session_num, out.memory_used); + printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n", + out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.session_num, out.memory_used); event_add((struct event *)arg, &tv); }