gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-5746 使用hos存储非结构化文件

Browse Source 
修复缓存代码编译问题
缓存功能适配hos
This commit is contained in:
fengweihao
2021-03-22 14:10:43 +08:00
parent 62dde9ea6e
commit 2b649f4510
13 changed files with 160 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
cache/src/tango_cache_transfer.cpp vendored
View File

@@ -114,7 +114,7 @@ static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, v
static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool full)
{
UNUSED CURLMcode rc;
char minio_url[256];
char minio_url[256], buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
@@ -136,8 +136,10 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
//token<65>ֶΣ<D6B6><CEA3><EFBFBD><EFBFBD><EFBFBD>hos<6F>洢<EFBFBD><E6B4A2>֤
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put.upload_length);
curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb);
@@ -178,9 +180,9 @@ static size_t curl_response_body_save_cb(void *ptr, size_t size, size_t count, v
int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
{
UNUSED CURLMcode rc;
char minio_url[256];
if(NULL == (ctx->curl=curl_easy_init()))
char minio_url[256]={0}, buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
return -1;
}
@@ -193,6 +195,8 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
@@ -205,7 +209,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
int cache_delete_minio_object(struct tango_cache_ctx *ctx, bool call_back)
{
UNUSED CURLMcode rc;
char minio_url[256];
char minio_url[256], buffer[256];
ctx->instance->statistic.del_recv_num += 1;
if(NULL == (ctx->curl=curl_easy_init()))
@@ -221,6 +225,9 @@ int cache_delete_minio_object(struct tango_cache_ctx *ctx, bool call_back)
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
@@ -257,9 +264,9 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
{
int len=0;
UNUSED CURLMcode rc;
char minio_url[256];
if(NULL == (ctx->curl=curl_easy_init()))
char minio_url[256], buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
return false;
}
@@ -282,6 +289,8 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
ctx->headers = NULL;
}
ctx->headers = curl_slist_append(ctx->headers, "Content-Type: application/xml");
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
@@ -298,7 +307,7 @@ bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block
switch(ctx->put.state)
{
case PUT_STATE_START:
if(sessions_exceeds_limit(ctx->instance, OBJECT_IN_MINIO))
if(sessions_exceeds_limit(ctx->instance, OBJECT_IN_HOS))
{
tango_cache_set_fail_state(ctx, CACHE_OUTOF_SESSION);
return false;
@@ -368,7 +377,7 @@ int do_tango_cache_update_end(struct tango_cache_ctx *ctx, bool callback)
tango_cache_ctx_destroy(ctx, callback);
return -1;
}
if(ctx->locate == OBJECT_IN_MINIO)
if(ctx->locate == OBJECT_IN_HOS)
{
return http_put_complete_part_evbuf(ctx, callback);
}
@@ -494,7 +503,7 @@ void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long r
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
}
if(ctx->instance->param->object_store_way!=CACHE_ALL_MINIO && !ctx->fail_state)
if(ctx->instance->param->object_store_way!=CACHE_ALL_HOS && !ctx->fail_state)
{
redis_put_minio_object_meta(ctx, true);
}
@@ -510,7 +519,7 @@ void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long r
int http_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback)
{
UNUSED CURLMcode rc;
char minio_url[256];
char minio_url[256], buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
@@ -527,6 +536,8 @@ int http_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COP
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
@@ -557,7 +568,7 @@ int do_tango_cache_upload_once_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY
ctx->instance->statistic.memory_used += size;
ctx->instance->error_code = CACHE_OK;
if(ctx->locate == OBJECT_IN_MINIO)
if(ctx->locate == OBJECT_IN_HOS)
{
return http_put_complete_part_data(ctx, way, data, size, false);
}
@@ -595,7 +606,7 @@ int do_tango_cache_upload_once_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_
size = evbuffer_get_length(ctx->put.evbuf);
ctx->instance->statistic.memory_used += size;
if(ctx->locate == OBJECT_IN_MINIO)
if(ctx->locate == OBJECT_IN_HOS)
{
return http_put_complete_part_evbuf(ctx, callback);
}
@@ -804,7 +815,7 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
{
return raw_len;
}
ctx->get.result.location = OBJECT_IN_MINIO;
ctx->get.result.location = OBJECT_IN_HOS;
}
pos_colon = (char*)memchr(start, ':', raw_len);
if(pos_colon == NULL)
@@ -893,7 +904,7 @@ void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long r
static int tango_cache_fetch_minio(struct tango_cache_ctx *ctx)
{
UNUSED CURLMcode rc;
char minio_url[256];
char minio_url[256], buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
@@ -912,6 +923,9 @@ static int tango_cache_fetch_minio(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_get_response_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
@@ -924,7 +938,7 @@ static void redis_redirect_object2minio_cb(struct tango_cache_ctx *ctx)
struct promise *p = ctx->promise;
ctx->get.state = GET_STATE_START;
ctx->locate = OBJECT_IN_MINIO;
ctx->locate = OBJECT_IN_HOS;
if(ctx->instance->statistic.session_http>=ctx->instance->param->maximum_sessions)
{
tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY);
@@ -942,8 +956,8 @@ int do_tango_cache_fetch_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATIO
ctx->instance->statistic.get_recv_num += 1;
switch(where_to_get)
{
case OBJECT_IN_MINIO:
ctx->locate = OBJECT_IN_MINIO;
case OBJECT_IN_HOS:
ctx->locate = OBJECT_IN_HOS;
return (tango_cache_fetch_minio(ctx)==1)?0:-2;
case OBJECT_IN_REDIS:
ctx->locate = OBJECT_IN_REDIS;