支持缓存校验。

2018-10-25 18:45:33 +08:00
parent 990e247155
commit 27d4581d9b
7 changed files with 523 additions and 211 deletions
--- a/plugin/business/pangu-http/src/pangu_web_cache.cpp
+++ b/plugin/business/pangu-http/src/pangu_web_cache.cpp
@@ -26,6 +26,7 @@ enum cache_stat_field
 	STAT_CACHE_QUERY_ERR,
 	STAT_CACHE_QUERY_ABANDON,
 	STAT_CACHE_QUERYING,
+	STAT_CACHE_META_QUERYING,
 	STAT_CACHE_UPLOAD_CNT,
 	STAT_CACHE_UPLOAD_OVERRIDE,
 	STAT_CACHE_UPLOAD_FORBIDEN,
@@ -269,12 +270,6 @@ static char* read_http1_hdr(const char* hdr, const char* field_name)
 	memcpy(value, p, q-p);
 	return value;
 }
-void cache_query_free_meta(struct cached_meta* meta)
-{
-	FREE(&meta->content_type);
-	FREE(&meta);
-	return;
-}

 enum cache_query_result_type cache_query_result_get_type(future_result_t * result)
 {
@@ -313,7 +308,10 @@ struct cache_query_context
 {
 	struct cache_handle* ref_handle;
 	char* url;
-	int is_undefined_obj;
+
+	struct cached_meta meta;
+	
+	struct tango_cache_result* ref_tango_cache_result;
 	struct future* f_tango_cache_fetch;
 };
 void cache_query_ctx_free_cb(void* p)
@@ -322,27 +320,52 @@ void cache_query_ctx_free_cb(void* p)
 	future_destroy(ctx->f_tango_cache_fetch);
 	ctx->f_tango_cache_fetch=NULL;
 	FREE(&(ctx->url));
+	FREE(&(ctx->meta.etag));
+	FREE(&(ctx->meta.last_modified));
+	FREE(&(ctx->meta.content_type));
 	free(ctx);
 }
-static void wrap_cache_query_on_succ(future_result_t * result, void * user)
+const struct cached_meta* cache_query_result_read_meta(future_result_t * result)
+{
+	struct cache_query_context* ctx=(struct cache_query_context*)result;
+	return &(ctx->meta);
+}
+void cached_meta_set(struct cached_meta* meta, enum CACHE_RESULT_TYPE type, const char* data_frag, size_t size)
+{
+	switch(type)
+	{
+		case RESULT_TYPE_HEADER:
+			meta->content_type=read_http1_hdr((const char*)data_frag, "content-type");
+			break;
+		case RESULT_TYPE_USERTAG:
+			meta->last_modified=read_http1_hdr(data_frag, "Last-Modified");
+			meta->etag=read_http1_hdr(data_frag, "etag");
+			break;
+		default:
+			assert(0);
+			break;
+	}
+	return;
+}
+
+static void cache_query_obj_on_succ(future_result_t * result, void * user)
 {
 	struct promise * p = (struct promise *) user;
 	struct cache_query_context* ctx=(struct cache_query_context*)promise_get_ctx(p);
 	struct tango_cache_result* _result=tango_cache_read_result(result);
-	enum cache_query_result_type type=cache_query_result_get_type(result);
 	
 	switch(_result->type)
 	{
 		case RESULT_TYPE_HEADER:
-			if(ctx->is_undefined_obj)
-			{
-				ATOMIC_INC(&(ctx->ref_handle->stat_val[STAT_CACHE_OVERRIDE_HIT]));
-				FS_operate(ctx->ref_handle->fs_handle, ctx->ref_handle->fs_id[STAT_CACHE_OVERRIDE_HIT_OBJ_SIZE], 0, FS_OP_SET, _result->tlength/1024);
-			}
 			ATOMIC_INC(&(ctx->ref_handle->stat_val[STAT_CACHE_QUERY_HIT]));
 			FS_operate(ctx->ref_handle->fs_handle, ctx->ref_handle->fs_id[STAT_CACHE_QUERY_HIT_OJB_SIZE], 0, FS_OP_SET, _result->tlength/1024);
+			cached_meta_set(&ctx->meta, RESULT_TYPE_HEADER, _result->data_frag, _result->size);
+			ctx->meta.content_length=_result->tlength;
 			TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache query hit: %s", ctx->url);
 			break;
+		case RESULT_TYPE_USERTAG:		
+			cached_meta_set(&ctx->meta, RESULT_TYPE_USERTAG, _result->data_frag, _result->size);
+			break;
 		case RESULT_TYPE_MISS:
 			TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache query miss: %s", ctx->url);
 			//NOT break intentionally.
@@ -362,7 +385,7 @@ static void wrap_cache_query_on_succ(future_result_t * result, void * user)

 	return;
 }
-static void wrap_cache_query_on_fail(enum e_future_error err, const char * what, void * user)
+static void cache_query_obj_on_fail(enum e_future_error err, const char * what, void * user)
 {
 	struct promise * p = (struct promise *) user;
 	struct cache_query_context* ctx=(struct cache_query_context*)promise_dettach_ctx(p);
@@ -372,45 +395,160 @@ static void wrap_cache_query_on_fail(enum e_future_error err, const char * what,
 	return;
 }

-enum cache_query_status async_web_cache_query(struct cache_handle* handle, unsigned int thread_id,
-										const struct tfe_http_half * request, struct future* f)
+
+struct cache_pending_context
 {
+	enum cache_pending_result status;
+	int is_undefined_obj;
 	struct request_freshness req_fresshness;
-	enum cache_pending_action get_action;	
-	struct cache_query_context* query_ctx=NULL;
-	struct promise* p=NULL;
-	struct future* _f=NULL;
-	int ret=0, is_undefined_obj=0;
-	get_action=tfe_cache_get_pending(request, &req_fresshness);
+	char* req_if_none_match, *req_if_modified_since;
+	const struct tfe_http_half * request;
+
+	char* url;
+	struct cached_meta cached_obj_meta;
+	
+	struct cache_handle* ref_handle;
+	struct tango_cache_result* ref_tango_cache_result;
+	struct future* f_tango_cache_fetch;
+};
+void cache_pending_ctx_free_cb(void* p)
+{
+	struct cache_pending_context* ctx=(struct cache_pending_context*)p;
+	ctx->request=NULL;
+	free(ctx->url);
+	free(ctx);
+	return;
+}
+const struct cached_meta* cache_pending_result_read_meta(future_result_t * result)
+{
+	struct cache_pending_context* ctx=(struct cache_pending_context*)result;
+	return &(ctx->cached_obj_meta);
+}
+
+static void cache_query_meta_on_succ(future_result_t * result, void * user)
+{
+	struct promise * p = (struct promise *) user;
+	struct cache_pending_context* ctx=(struct cache_pending_context*)promise_get_ctx(p);
+	struct tango_cache_result* _result=tango_cache_read_result(result);
+	ctx->ref_tango_cache_result=_result;
+	time_t cache_last_modified_time=0, request_last_modified_time=0;
+	switch(_result->type)
+	{
+		case RESULT_TYPE_HEADER:
+			ctx->cached_obj_meta.content_length=_result->tlength;
+			cached_meta_set(&ctx->cached_obj_meta, RESULT_TYPE_HEADER, _result->data_frag, _result->size);
+			ctx->status=PENDING_RESULT_REVALIDATE;
+			break;
+		case RESULT_TYPE_USERTAG:
+			cached_meta_set(&ctx->cached_obj_meta, RESULT_TYPE_HEADER, _result->data_frag, _result->size);
+			TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache meta query hit: %s %s %s"
+							, ctx->url
+							, ctx->cached_obj_meta.last_modified ? ctx->cached_obj_meta.last_modified:"no_last_modify"
+							, ctx->cached_obj_meta.etag ? ctx->cached_obj_meta.etag:"no_etag");
+			break;
+		case RESULT_TYPE_MISS:
+			ctx->status=PENDING_RESULT_MISS;
+			TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache meta query miss: %s", ctx->url);
+			//NOT break intentionally.
+		case RESULT_TYPE_END:
+			//last call.	
+			ATOMIC_DEC(&(ctx->ref_handle->stat_val[STAT_CACHE_META_QUERYING]));	
+			promise_dettach_ctx(p);		
+			promise_success(p, ctx);
+			cache_query_ctx_free_cb(ctx);
+			break;
+		default:
+			break;
+	}
+
+}
+static void cache_query_meta_on_fail(enum e_future_error err, const char * what, void * user)
+{
+	struct promise * p = (struct promise *) user;
+	struct cache_pending_context* ctx=(struct cache_pending_context*)promise_dettach_ctx(p);
+	promise_failed(p, err, what);	
+	ATOMIC_DEC(&(ctx->ref_handle->stat_val[STAT_CACHE_META_QUERYING]));	
+	cache_query_ctx_free_cb(ctx);
+	return;
+}
+
+enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id,
+												const struct tfe_http_half * request, struct future* f_revalidate)
+{	
+	struct request_freshness req_fresshness={0,0};
+	enum cache_pending_result result=PENDING_RESULT_FOBIDDEN;
+	int is_undefined_obj=0;
+	enum cache_pending_action get_action=tfe_cache_get_pending(request, &(req_fresshness));
 	switch(get_action)
 	{
 		case UNDEFINED:
 			if(!handle->query_undefined_obj_enabled)
 			{
 				ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_NOT_APPLICABLE]));
-				return WEB_CACHE_NOT_APPLICABLE;
+				result=PENDING_RESULT_FOBIDDEN;
+			}
+			else
+			{
+				is_undefined_obj=1;
+				ATOMIC_INC(&(handle->stat_val[STAT_CACHE_OVERRIDE_QUERY]));
+				result=PENDING_RESULT_ALLOWED;
 			}
-			is_undefined_obj=1;
-			ATOMIC_INC(&(handle->stat_val[STAT_CACHE_OVERRIDE_QUERY]));
 			break;
-		case VERIFY:
-			ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_VERIFY]));
-			return WEB_CACHE_NOT_APPLICABLE;
 		case FORBIDDEN:
 			ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_NOT_APPLICABLE]));
-			return WEB_CACHE_NOT_APPLICABLE;
+			result=PENDING_RESULT_FOBIDDEN;
+			break;
 		case ALLOWED:
+			result=PENDING_RESULT_ALLOWED;
 			break;
 		default:
-			assert(0);
-			return WEB_CACHE_NOT_APPLICABLE;
+			result=PENDING_RESULT_REVALIDATE;
+			break;
 	}
+	if(result!=PENDING_RESULT_REVALIDATE)
+	{
+		return result;
+	}
+	
+	struct tango_cache_meta_get meta;
+	memset(&meta, 0, sizeof(meta));
+	meta.url=request->req_spec.url;
+	meta.get=req_fresshness;
+	
+	struct promise* p=future_to_promise(f_revalidate);
+	struct cache_pending_context* ctx=ALLOC(struct cache_pending_context, 1);
+	ctx->status=PENDING_RESULT_FOBIDDEN;	
+	ctx->ref_handle=handle;
+	ctx->req_fresshness=req_fresshness;
+	ctx->url=tfe_strdup(request->req_spec.url);
+	ctx->req_if_modified_since=tfe_strdup(tfe_http_std_field_read(request, TFE_HTTP_IF_MODIFIED_SINCE));
+	ctx->req_if_none_match=tfe_strdup(tfe_http_std_field_read(request, TFE_HTTP_IF_NONE_MATCH));
+	promise_set_ctx(p, ctx, cache_pending_ctx_free_cb);
+
+	ATOMIC_INC(&(handle->stat_val[STAT_CACHE_META_QUERYING]));		
+	ctx->f_tango_cache_fetch=future_create("_cache_meta", cache_query_meta_on_succ, cache_query_meta_on_fail, p);
+	int ret=tango_cache_head_object(handle->clients[thread_id], ctx->f_tango_cache_fetch, &meta);
+	assert(ret==0);
+	return PENDING_RESULT_REVALIDATE;
+}
+int web_cache_async_query(struct cache_handle* handle, unsigned int thread_id,
+									const struct tfe_http_half * request, struct future* f)
+{
+	struct request_freshness req_fresshness;
+	enum cache_pending_action get_action;	
+	struct cache_query_context* query_ctx=NULL;
+	struct promise* p=NULL;
+	struct future* _f=NULL;
+
+	get_action=tfe_cache_get_pending(request, &req_fresshness);
+	assert(get_action!=FORBIDDEN);
+	
 	if(ATOMIC_READ(&(handle->stat_val[STAT_CACHE_QUERYING])) > ATOMIC_READ(&(handle->put_concurrency_max)))
 	{		
 		ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_ABANDON]));	
-		return WEB_CACHE_NOT_APPLICABLE;
+		return -1;
 	}
-	ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERYING]));	
+
 	struct tango_cache_meta_get meta;
 	memset(&meta, 0, sizeof(meta));
 	meta.url=request->req_spec.url;
@@ -418,14 +556,14 @@ enum cache_query_status async_web_cache_query(struct cache_handle* handle, unsig
 	query_ctx=ALLOC(struct cache_query_context, 1);
 	query_ctx->ref_handle=handle;
 	query_ctx->url=tfe_strdup(request->req_spec.url);
-	query_ctx->is_undefined_obj=is_undefined_obj;

 	p=future_to_promise(f);
 	promise_set_ctx(p, query_ctx, cache_query_ctx_free_cb);
-	query_ctx->f_tango_cache_fetch=future_create("_cache_get", wrap_cache_query_on_succ, wrap_cache_query_on_fail, p);
-	ret=tango_cache_fetch_object(handle->clients[thread_id], query_ctx->f_tango_cache_fetch, &meta);
-	assert(ret==0);
-	return WEB_CACHE_QUERING;
+	
+	ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERYING]));	
+	query_ctx->f_tango_cache_fetch=future_create("_cache_get", cache_query_obj_on_succ, cache_query_obj_on_fail, p);
+	int ret=tango_cache_fetch_object(handle->clients[thread_id], query_ctx->f_tango_cache_fetch, &meta);	
+	return 0;
 }
 struct wrap_cache_put_ctx
 {
@@ -460,20 +598,20 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle,
 	struct response_freshness resp_freshness;
 	enum cache_pending_action put_action;
 	struct tango_cache_ctx *write_ctx=NULL;
-	char buffer[TFE_STRING_MAX];
+	char cont_len_str[TFE_STRING_MAX]={0}, user_tag_str[TFE_STRING_MAX]={0};
 	const char* value=NULL;
+	char *tmp=NULL;
 	int i=0, is_undefined_obj=0;
 	size_t content_len=0;
-	if(!session->resp->resp_spec.content_length)
+	if(session->resp->resp_spec.content_length)
 	{
-		ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOAD_FORBIDEN]));
-		return NULL;
+		sscanf(session->resp->resp_spec.content_length, "%lu", &content_len);
 	}
-	sscanf(session->resp->resp_spec.content_length, "%lu", &content_len);
+	
 	put_action=tfe_cache_put_pending(session->resp, &resp_freshness);
 	switch(put_action){
 		case FORBIDDEN:
-		case VERIFY:
+		case REVALIDATE:
 			ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOAD_FORBIDEN]));
 			TFE_LOG_DEBUG(handle->logger, "cache update forbiden: %s", session->req->req_spec.url);
 			return NULL;
@@ -496,14 +634,26 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle,
 		ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOAD_ABANDON]));	
 		return NULL;
 	}
-	ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOADING]));		
+	ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOADING]));
+	
 	struct tango_cache_meta_put meta;
 	memset(&meta, 0, sizeof(meta));
 	meta.url=session->req->req_spec.url;
-	i=0;
-	snprintf(buffer, sizeof(buffer), "content-type:%s",session->resp->resp_spec.content_type);
-	meta.std_hdr[i]=buffer;
+	i=0;	
+
+	snprintf(cont_len_str, sizeof(cont_len_str), "content-type:%s",session->resp->resp_spec.content_type);
+	meta.std_hdr[i]=cont_len_str;
 	i++;
+	const char* etag=tfe_http_std_field_read(session->resp, TFE_HTTP_ETAG);
+	const char* last_modified=tfe_http_std_field_read(session->resp, TFE_HTTP_LAST_MODIFIED);
+	tmp=user_tag_str;
+	if(etag) tmp+=snprintf(tmp, sizeof(user_tag_str)-(tmp-user_tag_str), "etag:%s\r\n", etag);
+	if(last_modified) tmp+=snprintf(tmp, sizeof(user_tag_str)-(tmp-user_tag_str), "Last-modified:%s\r\n", last_modified);
+	if(strlen(user_tag_str)>0)
+	{
+		meta.usertag=user_tag_str;
+		meta.usertag_len=strlen(user_tag_str)+1;
+	}
 	memcpy(&meta.put, &resp_freshness, sizeof(resp_freshness));
 	
 	struct wrap_cache_put_ctx* _cache_put_ctx=ALLOC(struct wrap_cache_put_ctx, 1);