GET时HEADER只回调一次；增加CACHE MISS统计；

cache/Makefile

@@ -6,7 +6,7 @@ LDFLAGS =
 LIBS  = -lMESA_handle_logger -lMESA_htable -lMESA_prof_load -lwiredcfg
 LIBS += -lssl -lcrypto -lcurl -levent -lxml2
 
-OBJS = tfe_future.o tango_cache_tools.o tango_cache_client.o tango_cache_transfer.o tango_cache_xml.o cache_evbase_client.o
+OBJS = tango_cache_tools.o tango_cache_client.o tango_cache_transfer.o tango_cache_xml.o cache_evbase_client.o
 
 TARGET_A=pangu_tango_cache.a
 

cache/include/tango_cache_client.h

@@ -34,18 +34,19 @@ struct cache_statistics
 {
 	long long get_recv_num;	//<2F><><EFBFBD><EFBFBD>GET<45>Ĵ<EFBFBD><C4B4><EFBFBD>
 	long long get_succ_num;	//GET<45>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
-	long long get_fail_num;	//GETʧ<EFBFBD>ܵĴ<EFBFBD><EFBFBD><EFBFBD>
+	long long get_miss_num;	//GETδ<EFBFBD><EFBFBD><EFBFBD>еĴ<EFBFBD><EFBFBD><EFBFBD>
+	long long get_error_num;//GETʧ<54>ܵĴ<DCB5><C4B4><EFBFBD>
 	long long put_recv_num;	//<2F><><EFBFBD><EFBFBD>UPLOAD<41>Ĵ<EFBFBD><C4B4><EFBFBD>
 	long long put_succ_num;	//UPLOAD<41>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
-	long long put_fail_num;	//UPLOADʧ<44>ܵĴ<DCB5><C4B4><EFBFBD>
+	long long put_error_num;//UPLOADʧ<44>ܵĴ<DCB5><C4B4><EFBFBD>
 	long long memory_used;	//<2F><>ǰUPLOAD BODY<44><59>ռ<EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD>С
 	long long session_num;	//<2F><>ǰ<EFBFBD><C7B0><EFBFBD>ڽ<EFBFBD><DABD><EFBFBD>GET/PUT<55><54>HTTP<54>Ự<EFBFBD><E1BBB0>
 };
 
 enum CACHE_RESULT_TYPE
 {
-	RESULT_TYPE_HEADER=0,
+	RESULT_TYPE_HEADER=0, //ֻ<><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>
-	RESULT_TYPE_USERTAG,
+	RESULT_TYPE_USERTAG,  //ֻ<><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>
 	RESULT_TYPE_BODY,
 };
 
@@ -74,7 +75,7 @@ struct tango_cache_meta
 	const char* usertag; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD>GETʱ<54><CAB1>ԭ<EFBFBD><D4AD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
 	size_t usertag_len;  //<2F><><EFBFBD>󳤶<EFBFBD>USER_TAG_MAX_LEN<45><4E>0<EFBFBD><30>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7>
 	union{
-		struct response_freshness put;//TODO
+		struct response_freshness put;
 		struct request_freshness get;
 	};
 };

cache/tango_cache_client.cpp

@@ -46,10 +46,11 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str
 {
 	out->get_recv_num = instance->statistic.get_recv_num;
 	out->get_succ_num = instance->statistic.get_succ_num;
-	out->get_fail_num = instance->statistic.get_fail_num;
+	out->get_error_num= instance->statistic.get_error_num;
+	out->get_miss_num = instance->statistic.get_miss_num;
 	out->put_recv_num = instance->statistic.put_recv_num;
 	out->put_succ_num = instance->statistic.put_succ_num;
-	out->put_fail_num = instance->statistic.put_fail_num;
+	out->put_error_num= instance->statistic.put_error_num;
 	out->session_num  = instance->statistic.session_num;
 	out->memory_used  = instance->statistic.memory_used;
 }
@@ -64,13 +65,13 @@ const char *tango_cache_get_object_key(struct tango_cache_ctx *ctx)
 	return ctx->object_key;
 }
 
-static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, struct cache_statistics *statistic)
+static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, enum CACHE_ERR_CODE error_code, struct cache_statistics *statistic)
 {
 	if(method == CACHE_REQUEST_PUT)
 	{
 		if(fail_state)
 		{
-			statistic->put_fail_num += 1;
+			statistic->put_error_num += 1;
 		}
 		else
 		{
@@ -81,7 +82,10 @@ static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail
 	{
 		if(fail_state)
 		{
-			statistic->get_fail_num += 1;
+			if(error_code == CACHE_ERR_CURL)
+				statistic->get_error_num += 1;
+			else
+				statistic->get_miss_num += 1;
 		}
 		else
 		{
@@ -154,7 +158,7 @@ void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx)
 			}
 		}
 	}
-	update_statistics(ctx->method, ctx->fail_state, &ctx->instance->statistic);
+	update_statistics(ctx->method, ctx->fail_state, ctx->error_code, &ctx->instance->statistic);
 	free(ctx);
 }
 
@@ -265,6 +269,10 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
 			ctx->headers = curl_slist_append(ctx->headers, meta->std_hdr[i]);
 		}
 	}
+	if(meta->std_hdr[HDR_CONTENT_TYPE] == NULL)
+	{
+		ctx->headers = curl_slist_append(ctx->headers, "Content-Type:");
+	}
 	//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD>GETʱ<54><CAB1>ԭ<EFBFBD><D4AD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
 	if(meta->usertag_len>0 && meta->usertag_len<=USER_TAG_MAX_LEN)
 	{

cache/tango_cache_transfer.cpp

@@ -158,7 +158,7 @@ static size_t curl_write_uploadID_cb(void *ptr, size_t size, size_t count, void
 		if(code != CURLE_OK || ctx->res_code!=200L)
 		{
 			ctx->fail_state = true;
-			ctx->error_code = CACHE_CACHE_MISS;
+			ctx->error_code = CACHE_ERR_CURL;
 			if(code != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
 			return size*count;
 		}
@@ -339,6 +339,7 @@ int http_put_complete_part_evbuf(struct tango_cache_ctx *ctx)
 		if(ret <= 0)
 		{
 			ctx->fail_state = true;
+			ctx->error_code = CACHE_ERR_CURL;
 			tango_cache_ctx_destroy(ctx);
 		}
 	}
@@ -380,6 +381,7 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
 					else
 					{
 						ctx->fail_state = true;
+						ctx->error_code = CACHE_ERR_CURL;
 						tango_cache_ctx_destroy(ctx);
 					}
 				}
@@ -389,6 +391,7 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
 					if(ret <= 0)
 					{
 						ctx->fail_state = true;
+						ctx->error_code = CACHE_ERR_CURL;
 						if(cache_cancel_upload_minio(ctx))
 						{
 							ctx->put_state = PUT_STATE_CANCEL;
@@ -421,6 +424,10 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
 				ctx->error_code = CACHE_ERR_CURL;
 				ctx->fail_state = true;
 				if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
+				if(ctx->close_state)
+				{
+					tango_cache_ctx_destroy(ctx);
+				}
 			}
 			else
 			{
@@ -445,6 +452,7 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
 			if(res != CURLE_OK || res_code!=200L)
 			{
 				ctx->fail_state = true;
+				ctx->error_code = CACHE_ERR_CURL;
 				if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
 			}
 			if(ctx->fail_state)
@@ -483,6 +491,7 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
 			if(res != CURLE_OK || res_code!=200L)
 			{
 				ctx->fail_state = true;
+				ctx->error_code = CACHE_ERR_CURL;
 				if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
 			}
 			tango_cache_ctx_destroy(ctx);
@@ -577,7 +586,7 @@ void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
 			{
 				if(res!=CURLE_OK || res_code!=200L)
 				{
-					ctx->error_code = CACHE_ERR_CURL;
+					ctx->error_code = (res!=CURLE_OK)?CACHE_ERR_CURL:CACHE_CACHE_MISS;
 					promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, ctx->error);
 				}
 				else
@@ -619,17 +628,32 @@ static size_t curl_get_response_body_cb(void *ptr, size_t size, size_t count, vo
 	if(ctx->need_hdrs!=RESPONSE_HDR_ALL) //<2F><>Expiresʱ
 	{
 		ctx->fail_state = true;
-		ctx->error_code = CACHE_CACHE_MISS;
+		ctx->error_code = CACHE_ERR_CURL;
 		ctx->get_state	= GET_STATE_DELETE;
 		promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache Expires or last-modify not found");
+		return size*count;
 	}
-	else
+
+	if(ctx->response_tag.len > 0)
 	{
-		result.data_frag = ptr;
+		result.data_frag = ctx->response_tag.buff;
-		result.size = size * count;
+		result.size = ctx->response_tag.len;
-		result.type = RESULT_TYPE_BODY;
+		result.type = RESULT_TYPE_USERTAG;
 		promise_success(future_to_promise(ctx->future), &result);
+		easy_string_destroy(&ctx->response_tag);
 	}
+	if(ctx->response.len > 0)
+	{
+		result.data_frag = ctx->response.buff;
+		result.size = ctx->response.len;
+		result.type = RESULT_TYPE_HEADER;
+		promise_success(future_to_promise(ctx->future), &result);
+		easy_string_destroy(&ctx->response);
+	}
+	result.data_frag = ptr;
+	result.size = size * count;
+	result.type = RESULT_TYPE_BODY;
+	promise_success(future_to_promise(ctx->future), &result);
 	return size*count;
 }
 
@@ -654,7 +678,6 @@ static bool check_expires_header(struct tango_cache_ctx *ctx, const char *expire
 
 static bool check_fresh_header(struct tango_cache_ctx *ctx)
 {
-	struct tango_cache_result result;
 	time_t now_gmt;
 
 	if(ctx->need_hdrs != RESPONSE_HDR_ALL)
@@ -669,23 +692,6 @@ static bool check_fresh_header(struct tango_cache_ctx *ctx)
 		promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not fresh");
 		return false;
 	}
-
-	if(ctx->response_tag.len > 0)
-	{
-		result.data_frag = ctx->response_tag.buff;
-		result.size = ctx->response_tag.len;
-		result.type = RESULT_TYPE_USERTAG;
-		promise_success(future_to_promise(ctx->future), &result);
-		easy_string_destroy(&ctx->response_tag);
-	}
-	if(ctx->response.len > 0)
-	{
-		result.data_frag = ctx->response.buff;
-		result.size = ctx->response.len;
-		result.type = RESULT_TYPE_HEADER;
-		promise_success(future_to_promise(ctx->future), &result);
-		easy_string_destroy(&ctx->response);
-	}
 	return true;
 }
 
@@ -697,7 +703,7 @@ static bool check_get_result_code(struct tango_cache_ctx *ctx)
 	if(code != CURLE_OK || ctx->res_code!=200L)
 	{
 		ctx->fail_state = true;
-		ctx->error_code = CACHE_CACHE_MISS;
+		ctx->error_code = (code!=CURLE_OK)?CACHE_ERR_CURL:CACHE_CACHE_MISS;
 		promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, (code!=CURLE_OK)?ctx->error:"cache not hit");
 		return false;
 	}
@@ -707,9 +713,7 @@ static bool check_get_result_code(struct tango_cache_ctx *ctx)
 static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, void *userp)
 {
 	struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
-	struct tango_cache_result result;
+	char *start=(char *)ptr, *pos_colon;
-	char *start=(char *)ptr, *pos_colon, *hdrdata=(char*)ptr;
-	bool ptr_valid=false;
 	size_t raw_len = size*count, hdrlen=size*count;
 	char usertag[2048];
 
@@ -722,8 +726,6 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
 		return raw_len;
 	}
 
-	result.type = RESULT_TYPE_HEADER;
-
 	if((pos_colon=(char*)memchr(start, ':', raw_len))!=NULL)
 	{
 		size_t datalen = pos_colon - start;
@@ -755,41 +757,18 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
 				{
 					if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0)
 					{
-						if(ctx->need_hdrs == RESPONSE_HDR_ALL)
+						easy_string_savedata(&ctx->response_tag, usertag, hdrlen);
-						{
-							ptr_valid = true;
-							result.type = RESULT_TYPE_USERTAG;
-							hdrdata = usertag;
-						}
-						else
-						{
-							easy_string_savedata(&ctx->response_tag, usertag, hdrlen);
-						}
 					}
 				}
 				break;
-			case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) ptr_valid = true; break;
+			case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
-			case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) ptr_valid = true; break;
+			case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
-			case 14: if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) ptr_valid = true; break;
+			case 14: if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
-			case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) ptr_valid = true; break;
+			case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
-			case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) ptr_valid = true; break;
+			case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
 			default: break;
 		}
 	}
-
-	if(ptr_valid)
-	{
-		if(ctx->need_hdrs == RESPONSE_HDR_ALL)
-		{
-			result.data_frag = hdrdata;
-			result.size = hdrlen;
-			promise_success(future_to_promise(ctx->future), &result);
-		}
-		else if(result.type == RESULT_TYPE_HEADER)
-		{
-			easy_string_savedata(&ctx->response, hdrdata, hdrlen);
-		}
-	}
 	return raw_len;
 }
 

cache/test_demo/cache_evbase_test.cpp

@@ -248,8 +248,8 @@ int main(int argc, char **argv)
 
 	struct cache_statistics out;
 	cache_evbase_get_statistics(instance_asyn, &out);
-	printf("get_recv: %llu, get_succ: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n",
+	printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n",
-		out.get_recv_num, out.get_succ_num, out.get_fail_num, out.put_recv_num, out.put_succ_num, out.put_fail_num, out.session_num, out.memory_used);
+		out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.session_num, out.memory_used);
 	return 0;
 }
 

cache/test_demo/pangu_tg_cahce.conf

@@ -15,7 +15,7 @@ CACHE_BLOCK_MAX_SIZE=8192
 MAX_USED_MEMORY_SIZE_MB=5120
 
 #上传时Expires头部的过期时间，单位秒，最小60（1分钟）
-CACHE_DEFAULT_TTL_SECOND=60
+CACHE_DEFAULT_TTL_SECOND=3600
 
 #是否对对象的名称进行哈希，开启哈希有助于提高上传下载的速率
 CACHE_OBJECT_KEY_HASH_SWITCH=1

cache/test_demo/tango_cache_test.c

@@ -284,8 +284,8 @@ void timer_cb(evutil_socket_t fd, short what, void *arg)
 	}*/
 
 	tango_cache_get_statistics(tango_instance, &out);
-	printf("get_recv: %llu, get_succ: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n",
+	printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n",
-		out.get_recv_num, out.get_succ_num, out.get_fail_num, out.put_recv_num, out.put_succ_num, out.put_fail_num, out.session_num, out.memory_used);
+		out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.session_num, out.memory_used);
 
 	event_add((struct event *)arg, &tv);
 }