diff --git a/cache/Makefile b/cache/Makefile index 0a93a29..41b3a25 100644 --- a/cache/Makefile +++ b/cache/Makefile @@ -6,7 +6,7 @@ LDFLAGS = LIBS = -lMESA_handle_logger -lMESA_htable -lMESA_prof_load -lwiredcfg LIBS += -lssl -lcrypto -lcurl -levent -lxml2 -OBJS = tango_cache_tools.o tango_cache_client.o tango_cache_transfer.o tango_cache_xml.o cache_evbase_client.o +OBJS = tfe_future.o tango_cache_tools.o tango_cache_client.o tango_cache_transfer.o tango_cache_xml.o cache_evbase_client.o TARGET_A=pangu_tango_cache.a diff --git a/cache/cache_evbase_client.cpp b/cache/cache_evbase_client.cpp index 2e00a7e..d7b8482 100644 --- a/cache/cache_evbase_client.cpp +++ b/cache/cache_evbase_client.cpp @@ -39,7 +39,7 @@ struct databuffer enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn) { - return ctx_asyn->ctx->error_code; + return tango_cache_get_last_error(ctx_asyn->ctx); } enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance) { @@ -51,6 +51,11 @@ void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, s tango_cache_get_statistics(instance->instance, out); } +struct tango_cache_result *cache_evbase_read_result(void *promise_result) +{ + return tango_cache_read_result(promise_result); +} + const char *cache_evbase_get_object_key(struct cache_evbase_ctx *ctx_asyn) { return tango_cache_get_object_key(ctx_asyn->ctx); @@ -351,7 +356,7 @@ int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct } if(objectkey != NULL) { - snprintf(objectkey, keysize, "%s", ctx->file_key); + snprintf(objectkey, keysize, "%s", ctx->object_key); } ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx)); @@ -402,7 +407,7 @@ int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struc } if(objectkey != NULL) { - snprintf(objectkey, keysize, "%s", ctx->file_key); + snprintf(objectkey, keysize, "%s", ctx->object_key); } ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx)); diff --git a/cache/include/cache_evbase_client.h b/cache/include/cache_evbase_client.h index d42d7c8..eb81600 100644 --- a/cache/include/cache_evbase_client.h +++ b/cache/include/cache_evbase_client.h @@ -20,18 +20,21 @@ struct cache_evbase_ctx struct cache_evbase_instance *instance_asyn; }; +/*API的使用说明参考tango_cache_client.h*/ + enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn); enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance); void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out); + struct cache_evbase_instance *cache_evbase_instance_new(const char* profile_path, const char* section, void *runtimelog); -struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta); -int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size); -int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf); -void cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn); -const char *cache_evbase_get_object_key(struct cache_evbase_ctx *ctx_asyn); +//GET接口 +int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta); +struct tango_cache_result *cache_evbase_read_result(void *promise_result); + +//一次性上传接口 int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* future, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, struct tango_cache_meta *meta, @@ -41,7 +44,13 @@ int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struc struct tango_cache_meta *meta, char *objectkey, size_t keysize); -int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta); +//流式上传接口 +struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta); +int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size); +int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf); +void cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn); + +const char *cache_evbase_get_object_key(struct cache_evbase_ctx *ctx_asyn); #endif diff --git a/cache/include/tango_cache_client.h b/cache/include/tango_cache_client.h index 9d35811..9e5c0cb 100644 --- a/cache/include/tango_cache_client.h +++ b/cache/include/tango_cache_client.h @@ -7,6 +7,8 @@ #include "tfe_future.h" #include "proxy_cache.h" +#define USER_TAG_MAX_LEN 1518 + enum CACHE_ERR_CODE { CACHE_OK=0, @@ -43,13 +45,14 @@ struct cache_statistics enum CACHE_RESULT_TYPE { RESULT_TYPE_HEADER=0, + RESULT_TYPE_USERTAG, RESULT_TYPE_BODY, }; //promise_success的结果result struct tango_cache_result { - void *data_frag; //如果type为RESULT_TYPE_HEADER,内容会包含一个换行 + const void *data_frag; //如果type为RESULT_TYPE_HEADER,内容会包含一个换行 size_t size; enum CACHE_RESULT_TYPE type; }; @@ -67,8 +70,9 @@ enum CACHE_HTTP_HDR_TYPE struct tango_cache_meta { const char* url; //缓存:URL;非结构化日志:文件名 - const char* std_hdr[HDR_CONTENT_NUM]; //完整头部,如包含"Content-Type:",不要包含换行 - const char* other_hdr; //最大长度不能超过1535字节,GET时会原样返回 + const char* std_hdr[HDR_CONTENT_NUM]; //完整头部,如"Content-Type: text/html",不要包含换行;NULL表示没有该头部; + const char* usertag; //可以是任意内容,GET时会原样返回 + size_t usertag_len; //最大长度USER_TAG_MAX_LEN,0表示没有该头部 union{ struct response_freshness put;//TODO struct request_freshness get; @@ -86,14 +90,25 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str //每个监听线程创建一个instance struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,const char* profile_path, const char* section, void *runtimelog); + + //成功时回调promise_success, result为NULL时表示结束; //失败时回调promise_failed(仅一次),使用get_last_error获取错误码; //future不可以为NULL int tango_cache_fetch(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta); -/*UPLOAD接口的API*/ -//完整一次上传;若objectkey不为空,则输出对象的KEY,当CACHE_OBJECT_KEY_HASH_SWITCH=1开启对文件名哈希时有用 -//返回0表示成功,<0表示失败;下同 +//从promise_success的result参数提取结果;需要Caller调用tango_cache_result_release释放; +struct tango_cache_result *tango_cache_read_result(void *promise_result); + + + +/* UPLOAD接口的API + * 注意: UPLOAD接口的API,若future不为NULL,则在上传结束时会调用通知回调函数,否则不调用; + */ + +/*完整一次上传API*/ +//若objectkey不为空,则输出对象的KEY,当CACHE_OBJECT_KEY_HASH_SWITCH=1开启对文件名哈希时有用; +//返回值: 0-成功;<0失败;下同 int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct future* future, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, struct tango_cache_meta *meta, @@ -102,15 +117,15 @@ int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf, struct tango_cache_meta *meta, char *objectkey, size_t keysize); -//流式上传 -//若tango_cache_update_start返回NULL,调用tango_cache_ctx_error查看错误码是否是CACHE_OUTOF_MEMORY(正常情况下是) -//若future不为NULL,则在上传结束时会调用通知回调函数,否则不调用 +/*流式上传API*/ +//返回值: 若为NULL则表示创建失败,调用tango_cache_ctx_error查看错误码是否是CACHE_OUTOF_MEMORY(正常情况下是); struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta); +//返回值: 0-成功;<0失败,调用tango_cache_get_last_error查看错误码; int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data, size_t size); int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf); void tango_cache_update_end(struct tango_cache_ctx *ctx); -//获取对象key值;当CACHE_OBJECT_KEY_HASH_SWITCH=1开启对文件名哈希时有用 +//获取对象key值;当CACHE_OBJECT_KEY_HASH_SWITCH=1开启对URL/文件名哈希时有用 const char *tango_cache_get_object_key(struct tango_cache_ctx *ctx); #endif diff --git a/cache/tango_cache_client.cpp b/cache/tango_cache_client.cpp index 8bfefa1..2735ea2 100644 --- a/cache/tango_cache_client.cpp +++ b/cache/tango_cache_client.cpp @@ -54,9 +54,14 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str out->memory_used = instance->statistic.memory_used; } +struct tango_cache_result *tango_cache_read_result(void *promise_result) +{ + return (struct tango_cache_result *)promise_result; +} + const char *tango_cache_get_object_key(struct tango_cache_ctx *ctx) { - return ctx->file_key; + return ctx->object_key; } static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, struct cache_statistics *statistic) @@ -100,7 +105,7 @@ void easy_string_savedata(struct easy_string *estr, const char *data, size_t len if(estr->size-estr->len < len+1) { estr->size += len*4+1; - estr->buff = (char*)realloc(estr->buff,estr->size); + estr->buff = (char*)realloc(estr->buff, estr->size); } memcpy(estr->buff+estr->len, data, len); @@ -117,7 +122,11 @@ void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx) } easy_string_destroy(&ctx->response); - if(ctx->method == CACHE_REQUEST_PUT) + if(ctx->method == CACHE_REQUEST_GET) + { + easy_string_destroy(&ctx->response_tag); + } + else { struct multipart_etag_list *etag; @@ -208,7 +217,6 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance * { struct tango_cache_ctx *ctx; char buffer[256]={0}; - int other_len; time_t expires, now, last_modify; if((u_int64_t)instance->statistic.memory_used >= instance->cache_limit_size) @@ -227,11 +235,11 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance * if(instance->hash_object_key) { caculate_sha256(meta->url, strlen(meta->url), buffer, 72); - snprintf(ctx->file_key, 72, "%c%c/%c%c/%s", buffer[0], buffer[1], buffer[2], buffer[3], buffer+4); + snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", buffer[0], buffer[1], buffer[2], buffer[3], buffer+4); } else { - snprintf(ctx->file_key, 72, "%s", meta->url); + snprintf(ctx->object_key, 256, "%s", meta->url); } //Expires字段,用于缓存内部判定缓存是否超时 @@ -258,11 +266,11 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance * } } //其他定义的头部,GET时会原样返回 - if(meta->other_hdr!=NULL && (other_len = strlen(meta->other_hdr))<1535) + if(meta->usertag_len>0 && meta->usertag_len<=USER_TAG_MAX_LEN) { - char *p = (char *)malloc((other_len/3 + 1)*4 + 18); //计算编码后所需空间;18=17+1: 头部+字符串结束符 + char *p = (char *)malloc((meta->usertag_len/3 + 1)*4 + 18); //计算编码后所需空间;18=17+1: 头部+字符串结束符 memcpy(p, "x-amz-meta-user: ", 17); - Base64_EncodeBlock((unsigned char*)meta->other_hdr, other_len, (unsigned char*)p+17); + Base64_EncodeBlock((const unsigned char*)meta->usertag, meta->usertag_len, (unsigned char*)p+17); ctx->headers = curl_slist_append(ctx->headers, p); free(p); } @@ -297,7 +305,7 @@ int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct f } if(objectkey != NULL) { - snprintf(objectkey, keysize, "%s", ctx->file_key); + snprintf(objectkey, keysize, "%s", ctx->object_key); } return tango_cache_upload_once_start_data(ctx, way, data, size); @@ -315,7 +323,7 @@ int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct } if(objectkey != NULL) { - snprintf(objectkey, keysize, "%s", ctx->file_key); + snprintf(objectkey, keysize, "%s", ctx->object_key); } return tango_cache_upload_once_start_evbuf(ctx, way, evbuf); @@ -338,11 +346,11 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i if(instance->hash_object_key) { caculate_sha256(meta->url, strlen(meta->url), sha256, 72); - snprintf(ctx->file_key, 72, "%c%c/%c%c/%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4); + snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4); } else { - snprintf(ctx->file_key, 72, "%s", meta->url); + snprintf(ctx->object_key, 256, "%s", meta->url); } return ctx; diff --git a/cache/tango_cache_client_in.h b/cache/tango_cache_client_in.h index e8434a1..13470ea 100644 --- a/cache/tango_cache_client_in.h +++ b/cache/tango_cache_client_in.h @@ -54,7 +54,6 @@ struct tango_cache_instance time_t relative_ttl; //缓存的相对有效期 u_int64_t cache_limit_size; long max_cnn_host; - int host_index; u_int32_t block_len; //申请buffercache内存的缓存块大小,每次update块大小最好不要超过该值,否则会增加拷贝次数 u_int32_t upload_block_size; //minio分段上传块的最小长度 enum CACHE_ERR_CODE error_code; @@ -74,10 +73,8 @@ struct tango_cache_ctx struct curl_slist *headers; struct future* future; char error[CURL_ERROR_SIZE]; - char file_key[72]; + char object_key[256]; char hostport[24]; //相同ctx使用相同的IP,保证pipeline顺序性 - u_int32_t host_index; - bool fail_state; enum CACHE_REQUEST_METHOD method; enum CACHE_ERR_CODE error_code; @@ -89,6 +86,7 @@ struct tango_cache_ctx }; u_int32_t part_index; u_int32_t need_hdrs; //宏RESPONSE_HDR_ + bool fail_state; bool close_state; //主动被调用关闭 long res_code; time_t max_age;//Get @@ -101,6 +99,7 @@ struct tango_cache_ctx char *uploadID; char *combine_xml; struct easy_string response; + struct easy_string response_tag; TAILQ_HEAD(__etag_list_head, multipart_etag_list) cache_head; struct tango_cache_instance *instance; @@ -112,6 +111,7 @@ struct curl_socket_data }; void easy_string_savedata(struct easy_string *estr, const char *data, size_t len); +void easy_string_expand(struct easy_string *estr, size_t to_size); void easy_string_destroy(struct easy_string *estr); void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx); diff --git a/cache/tango_cache_transfer.cpp b/cache/tango_cache_transfer.cpp index 74799b7..31895b5 100644 --- a/cache/tango_cache_transfer.cpp +++ b/cache/tango_cache_transfer.cpp @@ -93,7 +93,7 @@ static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, v ctx->upload_offset += send_len; ctx->instance->statistic.memory_used -= send_len; - return len; + return send_len; } //return value: <0:fail; =0: not exec; >0: OK @@ -110,11 +110,11 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful ctx->upload_offset = 0; if(full) { - snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key); + snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key); } else { - snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ++ctx->part_index, ctx->uploadID); + snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ++ctx->part_index, ctx->uploadID); curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb); curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); } @@ -129,7 +129,7 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L); curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->upload_length); @@ -179,7 +179,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx) return -1; } - snprintf(minio_url, 256, "http://%s/%s/%s?uploads", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key); + snprintf(minio_url, 256, "http://%s/%s/%s?uploads", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key); curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); @@ -192,7 +192,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx) curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); assert(rc==CURLM_OK); @@ -209,7 +209,7 @@ bool cache_delete_minio_object(struct tango_cache_ctx *ctx) return false; } - snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key); + snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key); curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); @@ -237,7 +237,7 @@ bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx) return false; } - snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ctx->uploadID); + snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID); curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); @@ -267,7 +267,7 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx) } construct_complete_xml(ctx, &ctx->combine_xml, &len); - snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ctx->uploadID); + snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID); curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); @@ -507,7 +507,7 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM } ctx->put_state = PUT_STATE_END; - snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key); + snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L); @@ -519,7 +519,7 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); if(way == PUT_MEM_COPY) { @@ -670,7 +670,15 @@ static bool check_fresh_header(struct tango_cache_ctx *ctx) return false; } - if(ctx->response.buff != NULL) + if(ctx->response_tag.len > 0) + { + result.data_frag = ctx->response_tag.buff; + result.size = ctx->response_tag.len; + result.type = RESULT_TYPE_USERTAG; + promise_success(future_to_promise(ctx->future), &result); + easy_string_destroy(&ctx->response_tag); + } + if(ctx->response.len > 0) { result.data_frag = ctx->response.buff; result.size = ctx->response.len; @@ -703,7 +711,7 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, char *start=(char *)ptr, *pos_colon, *hdrdata=(char*)ptr; bool ptr_valid=false; size_t raw_len = size*count, hdrlen=size*count; - char usrhdr[2048]; + char usertag[2048]; if(ctx->fail_state || ctx->get_state==GET_STATE_DELETE) { @@ -714,6 +722,8 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, return raw_len; } + result.type = RESULT_TYPE_HEADER; + if((pos_colon=(char*)memchr(start, ':', raw_len))!=NULL) { size_t datalen = pos_colon - start; @@ -743,10 +753,18 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, case 15: if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15)) { - if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usrhdr, 2048))>0) + if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0) { - hdrdata = usrhdr; - ptr_valid = true; + if(ctx->need_hdrs == RESPONSE_HDR_ALL) + { + ptr_valid = true; + result.type = RESULT_TYPE_USERTAG; + hdrdata = usertag; + } + else + { + easy_string_savedata(&ctx->response_tag, usertag, hdrlen); + } } } break; @@ -761,14 +779,13 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, if(ptr_valid) { - if(ctx->need_hdrs==RESPONSE_HDR_ALL) + if(ctx->need_hdrs == RESPONSE_HDR_ALL) { result.data_frag = hdrdata; result.size = hdrlen; - result.type = RESULT_TYPE_HEADER; promise_success(future_to_promise(ctx->future), &result); } - else + else if(result.type == RESULT_TYPE_HEADER) { easy_string_savedata(&ctx->response, hdrdata, hdrlen); } @@ -787,7 +804,7 @@ int tango_cache_fetch_start(struct tango_cache_ctx *ctx) return -1; } - snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key); + snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL,1L); @@ -801,7 +818,7 @@ int tango_cache_fetch_start(struct tango_cache_ctx *ctx) curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); //ctx->error="Operation too slow. Less than 1024 bytes/sec transferred the last 3 seconds" curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); assert(rc==CURLM_OK); diff --git a/cache/test_demo/cache_evbase_test.cpp b/cache/test_demo/cache_evbase_test.cpp index 17a09b1..95b7c40 100644 --- a/cache/test_demo/cache_evbase_test.cpp +++ b/cache/test_demo/cache_evbase_test.cpp @@ -33,21 +33,25 @@ struct future_pdata void get_future_success(future_result_t* result, void * user) { - struct tango_cache_result *res = (struct tango_cache_result *)result; + struct tango_cache_result *res = cache_evbase_read_result(result); struct future_pdata *pdata = (struct future_pdata *)user; char buffer[1024]; if(res != NULL) { - if(res->type==RESULT_TYPE_BODY) + switch(res->type) { - fwrite(res->data_frag, res->size, 1, pdata->fp); - } - else - { - memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); - buffer[res->size] = '\0'; - printf("%s", buffer); + case RESULT_TYPE_BODY: + fwrite(res->data_frag, res->size, 1, pdata->fp); + break; + + case RESULT_TYPE_USERTAG: + case RESULT_TYPE_HEADER: + memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); + buffer[res->size] = '\0'; + printf("%s", buffer); + break; + default:break; } } else //结束 @@ -135,7 +139,7 @@ int main(int argc, char **argv) if(argc != 2 && argc!=3) { - printf("USGAE: %s [get_out_file_index]\n", argv[0]); + printf("USGAE: %s [get_out_file_index]\n", argv[0]); return -1; } if(argc==3) @@ -164,7 +168,8 @@ int main(int argc, char **argv) meta.url = filename_in; meta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype"; meta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip"; - meta.other_hdr = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n"; + meta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n"; + meta.usertag_len = strlen(meta.usertag); p = method; while(*p=='\r'||*p=='\n') p++; diff --git a/cache/test_demo/pangu_tg_cahce.conf b/cache/test_demo/pangu_tg_cahce.conf index fa7e465..c398565 100644 --- a/cache/test_demo/pangu_tg_cahce.conf +++ b/cache/test_demo/pangu_tg_cahce.conf @@ -15,7 +15,7 @@ CACHE_BLOCK_MAX_SIZE=8192 MAX_USED_MEMORY_SIZE_MB=5120 #涓婁紶鏃禘xpires澶撮儴鐨勮繃鏈熸椂闂达紝鍗曚綅绉掞紝鏈灏60锛1鍒嗛挓锛 -CACHE_DEFAULT_TTL_SECOND=3600 +CACHE_DEFAULT_TTL_SECOND=60 #鏄惁瀵瑰璞$殑鍚嶇О杩涜鍝堝笇锛屽紑鍚搱甯屾湁鍔╀簬鎻愰珮涓婁紶涓嬭浇鐨勯熺巼 CACHE_OBJECT_KEY_HASH_SWITCH=1 diff --git a/cache/test_demo/tango_cache_test.c b/cache/test_demo/tango_cache_test.c index 22f350a..4ed976c 100644 --- a/cache/test_demo/tango_cache_test.c +++ b/cache/test_demo/tango_cache_test.c @@ -46,21 +46,25 @@ struct future_pdata void get_future_success(future_result_t* result, void * user) { - struct tango_cache_result *res = (struct tango_cache_result *)result; + struct tango_cache_result *res = tango_cache_read_result(result); struct future_pdata *pdata = (struct future_pdata *)user; char buffer[1024]; if(res != NULL) { - if(res->type==RESULT_TYPE_BODY) + switch(res->type) { - fwrite(res->data_frag, res->size, 1, pdata->fp); - } - else - { - memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); - buffer[res->size] = '\0'; - printf("%s", buffer); + case RESULT_TYPE_BODY: + fwrite(res->data_frag, res->size, 1, pdata->fp); + break; + + case RESULT_TYPE_USERTAG: + case RESULT_TYPE_HEADER: + memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); + buffer[res->size] = '\0'; + printf("%s", buffer); + break; + default:break; } } else @@ -159,7 +163,8 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg) meta.url = s; meta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype"; meta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip"; - meta.other_hdr = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n"; + meta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n"; + meta.usertag_len = strlen(meta.usertag); while(*p=='\r'||*p=='\n')p++; if(*p=='\0') continue;