update数据增加evbuffer接口

2018-09-21 14:50:41 +08:00
parent 0bfd49194e
commit 3fe4625e04
10 changed files with 560 additions and 410 deletions
--- a/cache/tango_cache_transfer.cpp
+++ b/cache/tango_cache_transfer.cpp
@@ -21,21 +21,25 @@ size_t curl_response_any_cb(void *ptr, size_t size, size_t count, void *userp)

 static size_t curl_put_multipart_header_cb(void *ptr, size_t size, size_t count, void *userp)
 {
-	struct buffer_cache_list *list = (struct buffer_cache_list *)userp;
+	struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
 	size_t totallen = size*count;
 	char *start = (char *)ptr, *end = start + totallen;
+	struct multipart_etag_list *etag;

-	if(list->etag == NULL && !strncmp(start, "Etag:", totallen>5?5:totallen))
+	if(!strncmp(start, "Etag:", totallen>5?5:totallen))
 	{
 		start += 5; end -= 1; totallen -= 5;
 		while(totallen>0 && (*start==' ')) {start++; totallen--;}
 		while(totallen>0 && (*end=='\r'||*end=='\n')) {end--; totallen--;}
 		if(totallen > 0)
 		{
+			etag = (struct multipart_etag_list *)malloc(sizeof(struct multipart_etag_list));
 			totallen = end - start + 1;
-			list->etag = (char *)malloc(totallen + 1);
-			memcpy(list->etag, start, totallen);
-			*(list->etag + totallen) = '\0';
+			etag->etag = (char *)malloc(totallen + 1);
+			etag->part_number = ctx->part_index;
+			memcpy(etag->etag, start, totallen);
+			*(etag->etag + totallen) = '\0';
+			TAILQ_INSERT_TAIL(&ctx->cache_head, etag, node);
 		}
 	}
 	
@@ -64,99 +68,75 @@ static size_t curl_put_once_send_cb(void *ptr, size_t size, size_t count, void *
 	if(ctx->response.len >= ctx->response.size)
 	{
 		ctx->instance->statistic.memory_used -= ctx->response.size; //δʹ<CEB4><CAB9>cache buffer<65><72><EFBFBD>Լ<EFBFBD><D4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD><EFBFBD><EFBFBD>
-		if(ctx->way == PUT_ONCE_COPY)
-		{
-			response_buffer_destroy(&ctx->response);
-		}
-		else
-		{
-			ctx->response.buff = NULL;
-			ctx->response.len = ctx->response.size = 0;
-		}
+		easy_string_destroy(&ctx->response);
 	}
 	return len;
 }

 static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, void *userp)
 {
-	size_t len=0, needlen=size * count, remainlen;
-	struct buffer_cache_list *list = (struct buffer_cache_list *)userp;
-	struct cache_buffer *next_cache;
+	size_t len, space=size*count, send_len;
+	struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;

-	if(size==0 || count==0 || list->cache_cur==NULL)
+	if(size==0 || count==0 || ctx->upload_offset>=ctx->upload_length)
 	{
 		return 0;
 	}

-	while(len<needlen && list->cache_cur!=NULL)
+	len = ctx->upload_length - ctx->upload_offset;
+	if(len > space)
 	{
-		remainlen = list->cache_cur->len - list->cache_cur->off;
-		if(needlen-len >= remainlen)
-		{
-			memcpy((char*)ptr+len, list->cache_cur->buf+list->cache_cur->off, remainlen);
-			len += remainlen;
-			
-			next_cache = TAILQ_NEXT(list->cache_cur, node);
-			TAILQ_REMOVE(&list->cache_list, list->cache_cur, node);
-			buffer_cache_destroy(list->cache_cur, list->ctx->instance);
-			list->cache_cur = next_cache;
-		}
-		else
-		{
-			memcpy((char*)ptr+len, list->cache_cur->buf+list->cache_cur->off, needlen-len);
-			list->cache_cur->off += needlen-len;
-			len = needlen;
-		}
+		len = space;
 	}
+	send_len = evbuffer_remove(ctx->evbuffer, ptr, len);
+	assert(send_len>0);
+	ctx->upload_offset += send_len;
+	ctx->instance->statistic.memory_used -= send_len;
 	
 	return len;
 }

 //return value: <0:fail; =0: not exec; >0: OK
-int http_put_bodypart_request(struct tango_cache_ctx *ctx, struct buffer_cache_list *list, bool full)
+static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool full)
 {
 	CURLMcode rc;
 	char minio_url[256];

-	list->cache_cur = TAILQ_FIRST(&list->cache_list);
-	if(list->cache_cur == NULL)
-	{
-		return 0; //<2F>Ѿ<EFBFBD><D1BE>ϴ<EFBFBD><CFB4><EFBFBD>
-	}
-	if(NULL == (list->curl=curl_easy_init())) 
+	if(NULL == (ctx->curl=curl_easy_init())) 
 	{
 		return -1;
 	}

+	ctx->upload_offset = 0;
 	if(full)
 	{
 		snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
 	}
 	else
 	{
-		snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, list->part_number, ctx->uploadID);
-		curl_easy_setopt(list->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb);
-		curl_easy_setopt(list->curl, CURLOPT_HEADERDATA, list);
+		snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ++ctx->part_index, ctx->uploadID);
+		curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb);
+		curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
 	}
-	curl_easy_setopt(list->curl, CURLOPT_URL, minio_url);
-	curl_easy_setopt(list->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
-	curl_easy_setopt(list->curl, CURLOPT_NOSIGNAL, 1L);
-	curl_easy_setopt(list->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
-	curl_easy_setopt(list->curl, CURLOPT_WRITEDATA, list);
-	curl_easy_setopt(list->curl, CURLOPT_ERRORBUFFER, ctx->error);
-	curl_easy_setopt(list->curl, CURLOPT_PRIVATE, ctx);
-	curl_easy_setopt(list->curl, CURLOPT_FOLLOWLOCATION, 1L);
-	curl_easy_setopt(list->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); 
-	curl_easy_setopt(list->curl, CURLOPT_HTTPHEADER, ctx->headers_puts);
-	curl_easy_setopt(list->curl, CURLOPT_LOW_SPEED_TIME, 2L);
-	curl_easy_setopt(list->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
+	curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
+	curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
+	curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L);
+	curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
+	curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
+	curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error);
+	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
+	curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L);
+	curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); 
+	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
+	curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
+	curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);

-	curl_easy_setopt(list->curl, CURLOPT_UPLOAD, 1L);
-	curl_easy_setopt(list->curl, CURLOPT_INFILESIZE, list->length);
-	curl_easy_setopt(list->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb);
-	curl_easy_setopt(list->curl, CURLOPT_READDATA, list);
+	curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
+	curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->upload_length);
+	curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb);
+	curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);

-	rc = curl_multi_add_handle(ctx->instance->multi_hd, list->curl);
+	rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
 	assert(rc==CURLM_OK);
 	return 1;
 }
@@ -183,16 +163,8 @@ static size_t curl_write_uploadID_cb(void *ptr, size_t size, size_t count, void
 			return size*count;
 		}
 	}
-	
-	if(estr->size-estr->len < size*count+1)
-	{
-		estr->size += size*count*2+1;
-		estr->buff = (char*)realloc(estr->buff,estr->size);
-	}
-	
-	memcpy(estr->buff+estr->len,ptr,size*count);
-	estr->len+=size*count;
-	estr->buff[estr->len]='\0';
+
+	easy_string_savedata(estr, (const char*)ptr, size*count);
 	return size*count;
 }

@@ -218,7 +190,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
 	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
 	curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L);
 	curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); 
-	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers_puts);
+	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
 	curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
 	curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);

@@ -310,6 +282,11 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
 	curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->combine_xml);
 	curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, len); //<2F><><EFBFBD><EFBFBD>Content-Length

+	if(ctx->headers != NULL)
+	{
+		curl_slist_free_all(ctx->headers);
+		ctx->headers = NULL;
+	}
 	ctx->headers = curl_slist_append(ctx->headers, "Content-Type: application/xml");
 	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);

@@ -319,15 +296,10 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
 }

 //return value: true-<2D>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD><C2BC><EFBFBD>false-δ<><CEB4><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD>
-bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, struct buffer_cache_list *list)
+bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block_len)
 {
 	int ret = 1;

-	if(ctx->fail_state)
-	{
-		return false;
-	}
-
 	switch(ctx->put_state)
 	{
 		case PUT_STATE_START:
@@ -335,12 +307,11 @@ bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, struct buffe
 			ret = curl_get_minio_uploadID(ctx);
 			break;

-		case PUT_STATE_START_DONE:
 		case PUT_STATE_PART:
-			ret = http_put_bodypart_request(ctx, list, false);
-			if(ret > 0)
+			if(ctx->curl == NULL)
 			{
-				ctx->part_runing_num++;
+				ctx->upload_length = block_len;
+				ret = http_put_bodypart_request_evbuf(ctx, false);
 			}
 			break;
 			
@@ -356,6 +327,29 @@ bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, struct buffe
 	return true;
 }

+int http_put_complete_part_evbuf(struct tango_cache_ctx *ctx)
+{
+	int ret=0;
+	
+	ctx->put_state = PUT_STATE_END;
+	ctx->upload_length = evbuffer_get_length(ctx->evbuffer);
+	if(ctx->upload_length > 0)
+	{
+		ret = http_put_bodypart_request_evbuf(ctx, true);
+		if(ret <= 0)
+		{
+			ctx->fail_state = true;
+			tango_cache_ctx_destroy(ctx);
+		}
+	}
+	else
+	{
+		tango_cache_ctx_destroy(ctx);
+	}
+
+	return ret;
+}
+
 int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
 {
 	int ret = 0;
@@ -370,38 +364,14 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
 	switch(ctx->put_state)
 	{
 		case PUT_STATE_START:
-			ctx->put_state = PUT_STATE_END;
-			if(ctx->list_cur->cache_cur->len > 0)
-			{
-				TAILQ_INSERT_TAIL(&ctx->list_cur->cache_list, ctx->list_cur->cache_cur, node);
-				ctx->list_cur->length += ctx->list_cur->cache_cur->len;
-				ret = http_put_bodypart_request(ctx, ctx->list_cur, true);
-				if(ret <= 0)
-				{
-					tango_cache_ctx_destroy(ctx);
-				}
-			}
-			else
-			{
-				tango_cache_ctx_destroy(ctx);
-			}
+			http_put_complete_part_evbuf(ctx);
 			break;
-			
+
 		case PUT_STATE_PART:
-			if(ctx->list_cur->length + ctx->list_cur->cache_cur->len > 0)
+			if(ctx->curl == NULL)
 			{
-				TAILQ_INSERT_TAIL(&ctx->list_cur->cache_list, ctx->list_cur->cache_cur, node);
-				ctx->list_cur->length += ctx->list_cur->cache_cur->len;
-				ctx->list_cur->cache_cur = NULL;
-				TAILQ_INSERT_TAIL(&ctx->cache_head, ctx->list_cur, node);
-				cache_kick_upload_minio_multipart(ctx, ctx->list_cur);
-				ctx->list_cur = NULL;
-			}
-			else 
-			{
-				buffer_cache_list_destroy(ctx->list_cur, ctx);
-				ctx->list_cur = NULL;
-				if(ctx->part_runing_num==0) //<2F><>ȫ<EFBFBD><C8AB><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD>ɣ<EFBFBD><C9A3><EFBFBD><EFBFBD><EFBFBD>ENDʱ<44><CAB1><EFBFBD><EFBFBD><EFBFBD>ݿ<EFBFBD><DDBF><EFBFBD>
+				ctx->upload_length = evbuffer_get_length(ctx->evbuffer);
+				if(ctx->upload_length == 0)
 				{
 					if(cache_kick_combine_minio(ctx))
 					{
@@ -409,9 +379,26 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
 					}
 					else
 					{
+						ctx->fail_state = true;
 						tango_cache_ctx_destroy(ctx);
 					}
-				}	
+				}
+				else
+				{
+					ret = http_put_bodypart_request_evbuf(ctx, false);
+					if(ret <= 0)
+					{
+						ctx->fail_state = true;
+						if(cache_cancel_upload_minio(ctx))
+						{
+							ctx->put_state = PUT_STATE_CANCEL;
+						}
+						else
+						{
+							tango_cache_ctx_destroy(ctx);
+						}
+					}
+				}
 			}
 			break;
 			
@@ -425,93 +412,67 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)

 void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code)
 {
-	struct buffer_cache_list *list;
-
 	switch(ctx->put_state)
 	{
 		case PUT_STATE_WAIT_START:
-			ctx->curl = NULL;
-			ctx->res_code = 0;
-			curl_multi_remove_handle(ctx->instance->multi_hd, easy);
-			curl_easy_cleanup(easy);
 			if(res!=CURLE_OK||res_code!=200L|| ctx->fail_state || !parse_uploadID_xml(ctx->response.buff, ctx->response.len, &ctx->uploadID))
 			{
-				response_buffer_destroy(&ctx->response);
+				easy_string_destroy(&ctx->response);
 				ctx->error_code = CACHE_ERR_CURL;
 				ctx->fail_state = true;
 				if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
 			}
 			else
 			{
-				free(ctx->response.buff);
-				ctx->response.buff = NULL;
-				ctx->put_state = PUT_STATE_START_DONE;
-				TAILQ_FOREACH(list, &ctx->cache_head, node)
-				{
-					if(!cache_kick_upload_minio_multipart(ctx, list))
-					{
-						ctx->fail_state = true;
-						break;
-					}
-					else
-					{
-						ctx->put_state = PUT_STATE_PART;
-					}
-				}
-			}
-			if(ctx->close_state)
-			{
-				if(!ctx->fail_state)
+				easy_string_destroy(&ctx->response);
+				ctx->put_state = PUT_STATE_PART;
+				if(ctx->close_state)
 				{
 					cache_kick_upload_minio_end(ctx);
 				}
-				else if(ctx->put_state!=PUT_STATE_PART)
+				else
 				{
-					tango_cache_ctx_destroy(ctx);
+					size_t upload_length = evbuffer_get_length(ctx->evbuffer);
+					if(upload_length >= ctx->instance->upload_block_size)
+					{
+						cache_kick_upload_minio_multipart(ctx, upload_length);
+					}
 				}
 			}
 			break;

 		case PUT_STATE_PART:
-			curl_multi_remove_handle(ctx->instance->multi_hd, easy);
-			curl_easy_cleanup(easy);
-			ctx->part_runing_num--;
-			TAILQ_FOREACH(list, &ctx->cache_head, node)
-			{
-				if(list->curl == easy)
-				{
-					list->curl = NULL;
-					break;
-				}
-			}
-			assert(list != NULL); //PART״̬<D7B4><CCAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
-			if(res != CURLE_OK ||res_code!=200L )
+			if(res != CURLE_OK || res_code!=200L)
 			{
 				ctx->fail_state = true;
 				if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
 			}
-			if(ctx->part_runing_num==0 && ctx->list_cur==NULL)
+			if(ctx->fail_state)
 			{
-				if(ctx->fail_state && cache_cancel_upload_minio(ctx))
+				if(cache_cancel_upload_minio(ctx))
 				{
 					ctx->put_state = PUT_STATE_CANCEL;
 				}
-				else if(!ctx->fail_state && ctx->close_state && cache_kick_combine_minio(ctx))
-				{
-					ctx->put_state = PUT_STATE_END;
-				}
 				else if(ctx->close_state)
 				{
 					tango_cache_ctx_destroy(ctx);
 				}
 			}
+			else if(ctx->close_state)
+			{
+				cache_kick_upload_minio_end(ctx);
+			}
+			else
+			{
+				size_t upload_length = evbuffer_get_length(ctx->evbuffer);
+				if(upload_length >= ctx->instance->upload_block_size)
+				{
+					cache_kick_upload_minio_multipart(ctx, upload_length);
+				}
+			}
 			break;
 			
 		case PUT_STATE_CANCEL: //<2F>ȴ<EFBFBD><C8B4>ر<EFBFBD>
-			ctx->curl = NULL;
-			ctx->res_code = 0;
-			curl_multi_remove_handle(ctx->instance->multi_hd, easy);
-			curl_easy_cleanup(easy);
 			if(ctx->close_state)
 			{
 				tango_cache_ctx_destroy(ctx);
@@ -530,7 +491,7 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
 	}
 }

-int tango_cache_upload_once_start(struct tango_cache_ctx *ctx, const char *data, size_t size)
+int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size)
 {
 	CURLMcode rc;
 	char minio_url[256];
@@ -538,7 +499,7 @@ int tango_cache_upload_once_start(struct tango_cache_ctx *ctx, const char *data,
 	if(NULL == (ctx->curl=curl_easy_init())) 
 	{
 		tango_cache_ctx_destroy(ctx);
-		if(ctx->way == PUT_ONCE_FREE)
+		if(way == PUT_MEM_FREE)
 		{
 			free((void *)data);
 		}
@@ -556,11 +517,11 @@ int tango_cache_upload_once_start(struct tango_cache_ctx *ctx, const char *data,
 	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
 	curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L);
 	curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); 
-	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers_puts);
+	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
 	curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
 	curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);

-	if(ctx->way == PUT_ONCE_COPY)
+	if(way == PUT_MEM_COPY)
 	{
 		ctx->response.buff = (char *)malloc(size);
 		memcpy(ctx->response.buff, data, size);
@@ -582,12 +543,33 @@ int tango_cache_upload_once_start(struct tango_cache_ctx *ctx, const char *data,
 	return 0;
 }

+int tango_cache_upload_once_start_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf)
+{
+	size_t size;
+
+	ctx->evbuffer = evbuffer_new();
+	size = evbuffer_get_length(evbuf);
+	if(way == EVBUFFER_MOVE)
+	{
+		if(evbuffer_add_buffer(ctx->evbuffer, evbuf))
+		{
+			return -1;
+		}
+	}
+	else
+	{
+		if(evbuffer_add_buffer_reference(ctx->evbuffer, evbuf))
+		{
+			return -1;
+		}
+	}
+	ctx->instance->statistic.memory_used += size;
+
+	return http_put_complete_part_evbuf(ctx);
+}
+
 void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code)
 {
-	curl_multi_remove_handle(ctx->instance->multi_hd, easy);
-	curl_easy_cleanup(easy);
-	ctx->curl = NULL;
-
 	switch(ctx->get_state)
 	{
 		case GET_STATE_START:
@@ -634,12 +616,12 @@ static size_t curl_get_response_body_cb(void *ptr, size_t size, size_t count, vo
 		return size*count;
 	}
 	
-	if(!ctx->expire_comes) //<2F><>Expiresʱ
+	if(ctx->need_hdrs!=RESPONSE_HDR_ALL) //<2F><>Expiresʱ
 	{
 		ctx->fail_state = true;
 		ctx->error_code = CACHE_CACHE_MISS;
 		ctx->get_state	= GET_STATE_DELETE;
-		promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache Expires not found");
+		promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache Expires or last-modify not found");
 	}
 	else
 	{
@@ -651,31 +633,85 @@ static size_t curl_get_response_body_cb(void *ptr, size_t size, size_t count, vo
 	return size*count;
 }

+static bool check_expires_header(struct tango_cache_ctx *ctx, const char *expires_val, size_t len)
+{
+	time_t time_gmt;
+
+	ctx->expires = expires_hdr2timestamp(expires_val, len);
+	time_gmt = get_gmtime_timestamp(time(NULL));
+	
+	if(time_gmt > ctx->expires) //<2F><><EFBFBD><EFBFBD>ʧЧ<CAA7><D0A7>TODO relative_age<67>ĺ<EFBFBD><C4BA><EFBFBD><EFBFBD><EFBFBD>ɶ
+	{
+		ctx->fail_state = true;
+		ctx->error_code = CACHE_TIMEOUT;
+		ctx->get_state  = GET_STATE_DELETE; //<2F><><EFBFBD><EFBFBD>ʧЧʱ<D0A7><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+		easy_string_destroy(&ctx->response);
+		promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not fresh");
+		return false;
+	}
+	return true;
+}
+
+static bool check_fresh_header(struct tango_cache_ctx *ctx)
+{
+	struct tango_cache_result result;
+	time_t now_gmt;
+
+	if(ctx->need_hdrs != RESPONSE_HDR_ALL)
+		return true;
+
+	now_gmt = get_gmtime_timestamp(time(NULL));
+	if(ctx->last_modify+ctx->max_age > now_gmt || now_gmt+ctx->min_fresh>ctx->expires)
+	{
+		ctx->fail_state = true;
+		ctx->error_code = CACHE_TIMEOUT;
+		easy_string_destroy(&ctx->response);
+		promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not fresh");
+		return false;
+	}
+
+	if(ctx->response.buff != NULL)
+	{
+		result.data_frag = ctx->response.buff;
+		result.size = ctx->response.len;
+		result.type = RESULT_TYPE_HEADER;
+		promise_success(future_to_promise(ctx->future), &result);
+		easy_string_destroy(&ctx->response);
+	}
+	return true;
+}
+
+static bool check_get_result_code(struct tango_cache_ctx *ctx)
+{
+	CURLcode code;
+
+	code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code);
+	if(code != CURLE_OK || ctx->res_code!=200L)
+	{
+		ctx->fail_state = true;
+		ctx->error_code = CACHE_CACHE_MISS;
+		promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, (code!=CURLE_OK)?ctx->error:"cache not hit");
+		return false;
+	}
+	return true;
+}
+
 static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, void *userp)
 {
 	struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
-	CURLcode code;
 	struct tango_cache_result result;
 	char *start=(char *)ptr, *pos_colon, *hdrdata=(char*)ptr;
 	bool ptr_valid=false;
 	size_t raw_len = size*count, hdrlen=size*count;
+	char usrhdr[2048];

 	if(ctx->fail_state || ctx->get_state==GET_STATE_DELETE)
 	{
 		return raw_len;
 	}
-
-	if(ctx->res_code==0) //<2F>״<EFBFBD>Ӧ<EFBFBD><D3A6>ʱ<EFBFBD>ȿ<EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>200
+	if(ctx->res_code==0 && !check_get_result_code(ctx)) //<2F>״<EFBFBD>Ӧ<EFBFBD><D3A6>ʱ<EFBFBD>ȿ<EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>200
 	{
-		code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code);
-		if(code != CURLE_OK || ctx->res_code!=200L)
-		{
-			ctx->fail_state = true;
-			ctx->error_code = CACHE_CACHE_MISS;
-			promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not hit");
-			if(code != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
-			return raw_len;
-		}
+		return raw_len;
 	}

 	if((pos_colon=(char*)memchr(start, ':', raw_len))!=NULL)
@@ -686,39 +722,30 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
 			case 7:
 				if(strcmp_one_word_mesa_equal_len("expires", "EXPIRES", start, 7))
 				{
-					time_t expire = expires_hdr2timestamp(pos_colon + 1, raw_len - datalen - 1);
-					time_t time_gmt = get_gmtime_timestamp(time(NULL));
-					if(time_gmt + ctx->relative_age > expire) //<2F><><EFBFBD><EFBFBD>ʧЧ<CAA7><D0A7>TODO relative_age<67>ĺ<EFBFBD><C4BA><EFBFBD><EFBFBD><EFBFBD>ɶ
+					ctx->need_hdrs |= RESPONSE_HDR_EXPIRES;
+					if(!check_expires_header(ctx, pos_colon + 1, raw_len - datalen - 1) || !check_fresh_header(ctx))
 					{
-						ctx->fail_state = true;
-						ctx->error_code = CACHE_TIMEOUT;
-						if(time_gmt>=expire) ctx->get_state  = GET_STATE_DELETE; //<2F><><EFBFBD><EFBFBD>ʧЧʱ<D0A7><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
-						response_buffer_destroy(&ctx->response);
-						promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not fresh");
 						return raw_len;
 					}
-					else if(ctx->response.buff != NULL)
+				}
+				break;
+			case 13:
+				if(strcmp_one_word_mesa_equal_len("x-amz-meta-lm", "X-AMZ-META-LM", start, 13))
+				{
+					ctx->need_hdrs |= RESPONSE_HDR_LAST_MOD;
+					sscanf(pos_colon+1, "%lu", &ctx->last_modify);
+					if(!check_fresh_header(ctx))
 					{
-						result.data_frag = ctx->response.buff;
-						result.size = ctx->response.len;
-						result.type = RESULT_TYPE_HEADER;
-						promise_success(future_to_promise(ctx->future), &result);
-						response_buffer_destroy(&ctx->response);
+						return raw_len;
 					}
-					ctx->expire_comes = true;
 				}
 				break;
 			case 15:
 				if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15))
 				{
-					if(ctx->response.size-ctx->response.len < raw_len+1)
+					if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usrhdr, 2048))>0)
 					{
-						ctx->response.size += raw_len*8 + 1;
-						ctx->response.buff = (char*)realloc(ctx->response.buff, ctx->response.size);
-					}
-					if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)ctx->response.buff+ctx->response.len, ctx->response.size-ctx->response.len))>0)
-					{
-						hdrdata = ctx->response.buff+ctx->response.len;
+						hdrdata = usrhdr;
 						ptr_valid = true;
 					}
 				}
@@ -734,7 +761,7 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,

 	if(ptr_valid)
 	{
-		if(ctx->expire_comes)
+		if(ctx->need_hdrs==RESPONSE_HDR_ALL)
 		{
 			result.data_frag = hdrdata;
 			result.size = hdrlen;
@@ -743,14 +770,7 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
 		}
 		else
 		{
-			if(ctx->response.size-ctx->response.len < hdrlen+1)
-			{
-				ctx->response.size += hdrlen*8 + 1;
-				ctx->response.buff = (char*)realloc(ctx->response.buff, ctx->response.size);
-			}
-			memcpy(ctx->response.buff+ctx->response.len, hdrdata, hdrlen);
-			ctx->response.len += hdrlen;
-			ctx->response.buff[ctx->response.len] = '\0';
+			easy_string_savedata(&ctx->response, hdrdata, hdrlen);
 		}
 	}
 	return raw_len;