diff --git a/cache/src/tango_cache_client.cpp b/cache/src/tango_cache_client.cpp index c6fa89d..54dd8b7 100644 --- a/cache/src/tango_cache_client.cpp +++ b/cache/src/tango_cache_client.cpp @@ -261,12 +261,12 @@ int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data, { if(ctx->fail_state) { - return -1; + return 0; //TODO: 暂时忽略返回值!! } if(evbuffer_add(ctx->put.evbuf, data, size)) { tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY); - return -1; + return 0; } ctx->instance->statistic.memory_used += size; if(evbuffer_get_length(ctx->put.evbuf) >= ctx->instance->upload_block_size) @@ -282,7 +282,7 @@ int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COP if(ctx->fail_state) { - return -1; + return 0;//TODO: 暂时忽略返回值!! } size = evbuffer_get_length(evbuf); @@ -291,7 +291,7 @@ int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COP if(evbuffer_add_buffer(ctx->put.evbuf, evbuf)) { tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY); - return -1; + return 0; } } else @@ -299,7 +299,7 @@ int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COP if(evbuffer_add_buffer_reference(ctx->put.evbuf, evbuf)) { tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY); - return -1; + return 0; } } ctx->instance->statistic.memory_used += size; diff --git a/cache/src/tango_cache_client_in.h b/cache/src/tango_cache_client_in.h index a9b7688..123a84d 100644 --- a/cache/src/tango_cache_client_in.h +++ b/cache/src/tango_cache_client_in.h @@ -158,7 +158,6 @@ void easy_string_destroy(struct easy_string *estr); void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx, bool callback=true); void tango_cache_set_fail_state(struct tango_cache_ctx *ctx, enum CACHE_ERR_CODE error_code); -const char *tango_cache_errcode2str(enum CACHE_ERR_CODE err_code); const char *tango_cache_get_errstring(const struct tango_cache_ctx *ctx); struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_put *meta); diff --git a/cache/src/tango_cache_transfer.cpp b/cache/src/tango_cache_transfer.cpp index 20dd2f6..c404e77 100644 --- a/cache/src/tango_cache_transfer.cpp +++ b/cache/src/tango_cache_transfer.cpp @@ -13,6 +13,19 @@ #include "tango_cache_xml.h" #include "tango_cache_tools.h" +static inline void curl_set_common_options(CURL *curl, long transfer_timeout, char *errorbuf) +{ + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //测试发现多链接有某链接接收卡住的情况 + //ctx->error="Operation too slow. Less than 1024 bytes/sec transferred the last 3 seconds" + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 2L); + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); +} + //response body很短或不关心时 size_t curl_response_any_cb(void *ptr, size_t size, size_t count, void *userp) { @@ -119,23 +132,16 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); } curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); - curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); - curl_easy_setopt(ctx->curl, CURLOPT_TIMEOUT, ctx->instance->transfer_timeout); //测试发现有某链接接收卡住的情况 curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L); curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put.upload_length); curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb); curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx); + curl_set_common_options(ctx->curl, ctx->instance->transfer_timeout, ctx->error); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); assert(rc==CURLM_OK); @@ -183,17 +189,11 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx) curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, 0); //默认使用回调函数调用fread,测试发现关闭Expect时会导致卡在curl_multi_socket_action curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); - curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL,1L); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); + curl_set_common_options(ctx->curl, ctx->instance->transfer_timeout, ctx->error); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); assert(rc==CURLM_OK); @@ -217,14 +217,10 @@ int cache_delete_minio_object(struct tango_cache_ctx *ctx, bool call_back) snprintf(minio_url, 256, "http://%s/%s/%s", ctx->hostaddr, ctx->instance->bucketname, ctx->object_key); curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); - curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); + curl_set_common_options(ctx->curl, ctx->instance->transfer_timeout, ctx->error); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); assert(rc==CURLM_OK); @@ -245,14 +241,10 @@ bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx) snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->hostaddr, ctx->instance->bucketname, ctx->object_key, ctx->put.uploadID); curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); - curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); + curl_set_common_options(ctx->curl, ctx->instance->transfer_timeout, ctx->error); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); assert(rc==CURLM_OK); @@ -275,17 +267,13 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx) snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->hostaddr, ctx->instance->bucketname, ctx->object_key, ctx->put.uploadID); curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); - curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL,1L); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, len); //填充Content-Length curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->put.combine_xml); + curl_set_common_options(ctx->curl, ctx->instance->transfer_timeout, ctx->error); if(ctx->headers != NULL) { @@ -508,18 +496,11 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM snprintf(minio_url, 256, "http://%s/%s/%s", ctx->hostaddr, ctx->instance->bucketname, ctx->object_key); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); - curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); - curl_easy_setopt(ctx->curl, CURLOPT_TIMEOUT, ctx->instance->transfer_timeout); //测试发现有某链接接收卡住的情况 curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); + curl_set_common_options(ctx->curl, ctx->instance->transfer_timeout, ctx->error); if(way == PUT_MEM_COPY) { @@ -632,17 +613,11 @@ int tango_cache_multi_delete_start(struct tango_cache_ctx *ctx, bool callback) curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, ctx->response.size); //填充Content-Length,在CURLOPT_COPYPOSTFIELDS之前设置 curl_easy_setopt(ctx->curl, CURLOPT_COPYPOSTFIELDS, ctx->response.buff); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); + curl_set_common_options(ctx->curl, ctx->instance->transfer_timeout, ctx->error); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); assert(rc==CURLM_OK); @@ -878,20 +853,12 @@ int tango_cache_fetch_start(struct tango_cache_ctx *ctx) { curl_easy_setopt(ctx->curl, CURLOPT_NOBODY, 1L); } - curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); - curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL,1L); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_get_response_body_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); - curl_easy_setopt(ctx->curl, CURLOPT_TIMEOUT, ctx->instance->transfer_timeout); //测试发现有某链接接收卡住的情况 curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_get_response_header_cb); curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); - //ctx->error="Operation too slow. Less than 1024 bytes/sec transferred the last 3 seconds" - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L); - curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); + curl_set_common_options(ctx->curl, ctx->instance->transfer_timeout, ctx->error); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); assert(rc==CURLM_OK); diff --git a/cache/test/cache_evbase_test_threads.cpp b/cache/test/cache_evbase_test_threads.cpp index 9d3827b..b792bea 100644 --- a/cache/test/cache_evbase_test_threads.cpp +++ b/cache/test/cache_evbase_test_threads.cpp @@ -193,7 +193,6 @@ void* thread_upload_download(void *arg) { sprintf(filename_out, "file_index_%d_%d.bin", thread_data->thread_id, i); pdata->future = future_create(get_future_success, get_future_failed, pdata); - promise_set_ctx(future_to_promise(pdata->future), NULL, NULL); pdata->fp = fopen(filename_out, "w"); cache_evbase_fetch_object(instance_asyn, pdata->future, &getmeta); @@ -201,7 +200,6 @@ void* thread_upload_download(void *arg) else if(!strcasecmp(p, "DEL")) { pdata->future = future_create(del_future_success, del_future_failed, pdata); - promise_set_ctx(future_to_promise(pdata->future), NULL, NULL); sprintf(pdata->filename, "%s", filename_in); cache_evbase_delete_object(instance_asyn, pdata->future, filename_in); } @@ -210,7 +208,6 @@ void* thread_upload_download(void *arg) size_t filelen; p = get_file_content(filename_in, &filelen); pdata->future = future_create(put_future_success, put_future_failed, pdata); - promise_set_ctx(future_to_promise(pdata->future), NULL, NULL); if(cache_evbase_upload_once_data(instance_asyn, pdata->future, PUT_MEM_FREE, p, filelen, &putmeta, pdata->filename, 256)) { @@ -223,7 +220,6 @@ void* thread_upload_download(void *arg) { size_t readlen; pdata->future = future_create(put_future_success, put_future_failed, pdata); - promise_set_ctx(future_to_promise(pdata->future), NULL, NULL); struct evbuffer *evbuf = evbuffer_new(); char buffer[1024]; @@ -249,7 +245,6 @@ void* thread_upload_download(void *arg) else { pdata->future = future_create(put_future_success, put_future_failed, pdata); - promise_set_ctx(future_to_promise(pdata->future), NULL, NULL); ctx = cache_evbase_update_start(instance_asyn, pdata->future, &putmeta); if(ctx==NULL) diff --git a/cache/test/lib/libhiredis.a b/cache/test/lib/libhiredis.a index 670ea58..e0c95b6 100644 Binary files a/cache/test/lib/libhiredis.a and b/cache/test/lib/libhiredis.a differ diff --git a/cache/test/tango_cache_test.c b/cache/test/tango_cache_test.c index 560673c..19c6b19 100644 --- a/cache/test/tango_cache_test.c +++ b/cache/test/tango_cache_test.c @@ -312,7 +312,7 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg) ctx = tango_cache_update_start(tango_instance, pdata->future, &putmeta); if(ctx==NULL) { - put_future_failed(FUTURE_ERROR_CANCEL, "NULL", pdata); + put_future_failed(FUTURE_ERROR_CANCEL, "tango_cache_update_start_NULL", pdata); continue; } tango_cache_get_object_path(ctx, pdata->filename, 256);