gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

对外API增加USERTAG

Browse Source
This commit is contained in:
zhangchengwei
2018-09-23 15:35:13 +08:00
parent 8d71ab8173
commit 20045a7493
10 changed files with 144 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
cache/tango_cache_transfer.cpp vendored
View File

@@ -93,7 +93,7 @@ static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, v
ctx->upload_offset += send_len;
ctx->instance->statistic.memory_used -= send_len;
return len;
return send_len;
}
//return value: <0:fail; =0: not exec; >0: OK
@@ -110,11 +110,11 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
ctx->upload_offset = 0;
if(full)
{
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
}
else
{
snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ++ctx->part_index, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ++ctx->part_index, ctx->uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
}
@@ -129,7 +129,7 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->upload_length);
@@ -179,7 +179,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/%s?uploads", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s?uploads", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -192,7 +192,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
@@ -209,7 +209,7 @@ bool cache_delete_minio_object(struct tango_cache_ctx *ctx)
return false;
}
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -237,7 +237,7 @@ bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx)
return false;
}
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -267,7 +267,7 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
}
construct_complete_xml(ctx, &ctx->combine_xml, &len);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -507,7 +507,7 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM
}
ctx->put_state = PUT_STATE_END;
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L);
@@ -519,7 +519,7 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
if(way == PUT_MEM_COPY)
{
@@ -670,7 +670,15 @@ static bool check_fresh_header(struct tango_cache_ctx *ctx)
return false;
}
if(ctx->response.buff != NULL)
if(ctx->response_tag.len > 0)
{
result.data_frag = ctx->response_tag.buff;
result.size = ctx->response_tag.len;
result.type = RESULT_TYPE_USERTAG;
promise_success(future_to_promise(ctx->future), &result);
easy_string_destroy(&ctx->response_tag);
}
if(ctx->response.len > 0)
{
result.data_frag = ctx->response.buff;
result.size = ctx->response.len;
@@ -703,7 +711,7 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
char *start=(char *)ptr, *pos_colon, *hdrdata=(char*)ptr;
bool ptr_valid=false;
size_t raw_len = size*count, hdrlen=size*count;
char usrhdr[2048];
char usertag[2048];
if(ctx->fail_state || ctx->get_state==GET_STATE_DELETE)
{
@@ -714,6 +722,8 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
return raw_len;
}
result.type = RESULT_TYPE_HEADER;
if((pos_colon=(char*)memchr(start, ':', raw_len))!=NULL)
{
size_t datalen = pos_colon - start;
@@ -743,10 +753,18 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
case 15:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15))
{
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usrhdr, 2048))>0)
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0)
{
hdrdata = usrhdr;
ptr_valid = true;
if(ctx->need_hdrs == RESPONSE_HDR_ALL)
{
ptr_valid = true;
result.type = RESULT_TYPE_USERTAG;
hdrdata = usertag;
}
else
{
easy_string_savedata(&ctx->response_tag, usertag, hdrlen);
}
}
}
break;
@@ -761,14 +779,13 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
if(ptr_valid)
{
if(ctx->need_hdrs==RESPONSE_HDR_ALL)
if(ctx->need_hdrs == RESPONSE_HDR_ALL)
{
result.data_frag = hdrdata;
result.size = hdrlen;
result.type = RESULT_TYPE_HEADER;
promise_success(future_to_promise(ctx->future), &result);
}
else
else if(result.type == RESULT_TYPE_HEADER)
{
easy_string_savedata(&ctx->response, hdrdata, hdrlen);
}
@@ -787,7 +804,7 @@ int tango_cache_fetch_start(struct tango_cache_ctx *ctx)
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL,1L);
@@ -801,7 +818,7 @@ int tango_cache_fetch_start(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
//ctx->error="Operation too slow. Less than 1024 bytes/sec transferred the last 3 seconds"
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);