gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【1】修复HTTP Expect头部缺失时POST卡顿的问题；

Browse Source 
【2】TODO：尝试增加multiple delete objects API，尚未成功（AccessDenied）；
This commit is contained in:
zhangchengwei
2018-10-11 16:35:13 +08:00
committed by zhengchao
parent 68ee4bdd79
commit a2fa705fa7
8 changed files with 308 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
cache/tango_cache_transfer.cpp vendored
View File

@@ -141,7 +141,7 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
return 1;
}
static size_t curl_write_uploadID_cb(void *ptr, size_t size, size_t count, void *userp)
static size_t curl_response_body_save_cb(void *ptr, size_t size, size_t count, void *userp)
{
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
struct easy_string *estr = &ctx->response;
@@ -158,7 +158,6 @@ static size_t curl_write_uploadID_cb(void *ptr, size_t size, size_t count, void
if(code != CURLE_OK || ctx->res_code!=200L)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
if(code != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
return size*count;
}
}
@@ -180,11 +179,12 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
snprintf(minio_url, 256, "http://%s/%s/%s?uploads", ctx->hostaddr, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, 0); //Ĭ<><C4AC>ʹ<EFBFBD>ûص<C3BB><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>fread<61><64><EFBFBD><EFBFBD><EFBFBD>Է<EFBFBD><D4B7>ֹر<D6B9>Expectʱ<74>ᵼ<EFBFBD>¿<EFBFBD><C2BF><EFBFBD>curl_multi_socket_action
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL,1L);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_write_uploadID_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
@@ -280,8 +280,8 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->put.combine_xml);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, len); //<2F><><EFBFBD><EFBFBD>Content-Length
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->put.combine_xml);
if(ctx->headers != NULL)
{
@@ -419,7 +419,6 @@ void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long r
{
easy_string_destroy(&ctx->response);
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
if(ctx->put.close_state)
{
tango_cache_ctx_destroy(ctx);
@@ -448,7 +447,6 @@ void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long r
if(res != CURLE_OK || res_code!=200L)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
}
if(ctx->fail_state)
{
@@ -486,7 +484,6 @@ void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long r
if(res != CURLE_OK || res_code!=200L)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
}
tango_cache_ctx_destroy(ctx);
break;
@@ -579,11 +576,77 @@ void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long r
{
if(res!=CURLE_OK || (res_code!=204L && res_code!=200L ))
{
ctx->fail_state = true;
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
}
tango_cache_ctx_destroy(ctx);
}
void tango_cache_curl_muldel_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
u_int32_t errnum=0;
if(res!=CURLE_OK || (res_code!=204L && res_code!=200L ))
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
ctx->del.fail_num = ctx->del.succ_num;
ctx->del.succ_num = 0;
}
else
{
if(!parse_multidelete_xml(ctx->response.buff, ctx->response.len, &errnum, ctx->error, CURL_ERROR_SIZE))
{
ctx->del.fail_num = ctx->del.succ_num;
ctx->del.succ_num = 0;
}
else
{
ctx->del.fail_num = errnum;
ctx->del.succ_num -= errnum;
}
if(ctx->del.fail_num > 0)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
}
}
tango_cache_ctx_destroy(ctx);
}
int tango_cache_multi_delete_start(struct tango_cache_ctx *ctx)
{
CURLMcode rc;
char minio_url[256];
ctx->instance->statistic.del_recv_num += ctx->del.succ_num;
ctx->instance->error_code = CACHE_OK;
if(NULL == (ctx->curl=curl_easy_init()))
{
tango_cache_ctx_destroy(ctx);
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/?delete", ctx->hostaddr, ctx->instance->bucketname);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, ctx->response.size); //<2F><><EFBFBD><EFBFBD>Content-Length<74><68><EFBFBD><EFBFBD>CURLOPT_COPYPOSTFIELDS֮ǰ<D6AE><C7B0><EFBFBD><EFBFBD>
curl_easy_setopt(ctx->curl, CURLOPT_COPYPOSTFIELDS, ctx->response.buff);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_ERRORBUFFER, ctx->error);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
easy_string_destroy(&ctx->response);
return 0;
}
void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
switch(ctx->get.state)