gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

将bucketname作为object_key的一部分；修改delete接口加入minioaddr和bucket可选参数。

Browse Source
This commit is contained in:
zhangchengwei
2018-12-15 13:50:48 +08:00
committed by zhengchao
parent 1059af3c72
commit a70aa35a3a
8 changed files with 56 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cache/src/cache_evbase_client.cpp vendored
View File

@@ -554,14 +554,14 @@ int cache_evbase_head_object(struct cache_evbase_instance *instance, struct futu
return 0;
}
int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* f, const char *objkey)
int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket)
{
struct cache_evbase_ctx *ctx_asyn;
struct databuffer *buffer;
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = tango_cache_delete_prepare(instance->instance, f, objkey);
ctx_asyn->ctx = tango_cache_delete_prepare(instance->instance, f, objkey, minio_addr, bucket);
if(ctx_asyn->ctx == NULL)
{
free(ctx_asyn);

30
cache/src/tango_cache_client.cpp vendored
View File

@@ -283,11 +283,11 @@ void tango_cache_get_object_path(struct tango_cache_ctx *ctx, char *path/*OUT*/,
{
if(ctx->locate == OBJECT_IN_MINIO)
{
snprintf(path, pathsize, "http://%s/%s/%s", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key);
snprintf(path, pathsize, "http://%s/%s", ctx->hostaddr, ctx->object_key);
}
else
{
snprintf(path, pathsize, "redis://%s/%s/%s", ctx->instance->redisaddr, ctx->instance->param->bucketname, ctx->object_key);
snprintf(path, pathsize, "redis://%s/%s", ctx->instance->redisaddr, ctx->object_key);
}
}
}
@@ -405,14 +405,14 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
if(instance->param->hash_object_key)
{
caculate_sha256(meta->url, strlen(meta->url), buffer, 72);
snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", buffer[0], buffer[1], buffer[2], buffer[3], buffer+4);
snprintf(ctx->object_key, 256, "%s/%c%c/%c%c/%s", instance->param->bucketname, buffer[0], buffer[1], buffer[2], buffer[3], buffer+4);
//<2F><><EFBFBD><EFBFBD>ԭʼURL
snprintf(buffer, 2064, "x-amz-meta-url: %s", meta->url);
ctx->headers = curl_slist_append(ctx->headers, buffer);
}
else
{
snprintf(ctx->object_key, 256, "%s", meta->url);
snprintf(ctx->object_key, 256, "%s/%s", instance->param->bucketname, meta->url);
}
if(wired_load_balancer_lookup(instance->param->minio.wiredlb, ctx->object_key, strlen(ctx->object_key), ctx->hostaddr, 48))
{
@@ -591,11 +591,11 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i
if(instance->param->hash_object_key)
{
caculate_sha256(meta->url, strlen(meta->url), sha256, 72);
snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
snprintf(ctx->object_key, 256, "%s/%c%c/%c%c/%s", instance->param->bucketname, sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
}
else
{
snprintf(ctx->object_key, 256, "%s", meta->url);
snprintf(ctx->object_key, 256, "%s/%s", instance->param->bucketname, meta->url);
}
if(wired_load_balancer_lookup(instance->param->minio.wiredlb, ctx->object_key, strlen(ctx->object_key), ctx->hostaddr, 48))
{
@@ -639,10 +639,11 @@ int tango_cache_head_object(struct tango_cache_instance *instance, struct future
return do_tango_cache_head_object(ctx, location);
}
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* f, const char *objkey)
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket)
{
struct tango_cache_ctx *ctx;
char sha256[72];
const char *pbucket;
if(sessions_exceeds_limit(instance, OBJECT_IN_MINIO))
{
@@ -656,16 +657,21 @@ struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *
ctx->promise = future_to_promise(f);
ctx->method = CACHE_REQUEST_DELETE;
pbucket = (bucket==NULL)?instance->param->bucketname:bucket;
if(instance->param->hash_object_key)
{
caculate_sha256(objkey, strlen(objkey), sha256, 72);
snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
snprintf(ctx->object_key, 256, "%s/%c%c/%c%c/%s", pbucket, sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
}
else
{
snprintf(ctx->object_key, 256, "%s", objkey);
snprintf(ctx->object_key, 256, "%s/%s", pbucket, objkey);
}
if(wired_load_balancer_lookup(instance->param->minio.wiredlb, ctx->object_key, strlen(ctx->object_key), ctx->hostaddr, 48))
if(minio_addr != NULL)
{
snprintf(ctx->hostaddr, 48, "%s", minio_addr);
}
else if(wired_load_balancer_lookup(instance->param->minio.wiredlb, ctx->object_key, strlen(ctx->object_key), ctx->hostaddr, 48))
{
instance->error_code = CACHE_ERR_WIREDLB;
instance->statistic.totaldrop_num += 1;
@@ -675,11 +681,11 @@ struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *
return ctx;
}
int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* f, const char *objkey)
int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket)
{
struct tango_cache_ctx *ctx;
ctx = tango_cache_delete_prepare(instance, f, objkey);
ctx = tango_cache_delete_prepare(instance, f, objkey, minio_addr, bucket);
if(ctx == NULL)
{
return -1;

4
cache/src/tango_cache_client_in.h vendored
View File

@@ -194,8 +194,8 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
struct future* f, struct tango_cache_meta_put *meta, enum OBJECT_LOCATION maybe_loc);
struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance,
enum CACHE_REQUEST_METHOD method, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get);
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* f, const char *objkey);
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance,
struct future* f, const char *objkey, const char *minio_addr, const char *bucket);
enum OBJECT_LOCATION tango_cache_object_locate(struct tango_cache_instance *instance, size_t object_size);
void tango_cache_get_object_path(struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize);

24
cache/src/tango_cache_redis.cpp vendored
View File

@@ -221,8 +221,8 @@ int tango_cache_head_redis(struct tango_cache_ctx *ctx)
{
int ret = -1;
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx, "HMGET %s/%s OBJECT_META OBJECT_LOCATION",
ctx->instance->param->bucketname, ctx->object_key);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx,
"HMGET %s OBJECT_META OBJECT_LOCATION", ctx->object_key);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
@@ -240,8 +240,8 @@ int tango_cache_fetch_redis(struct tango_cache_ctx *ctx)
{
int ret = -1;
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx, "HMGET %s/%s OBJECT_META OBJECT_LOCATION OBJECT_BODY",
ctx->instance->param->bucketname, ctx->object_key);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx,
"HMGET %s OBJECT_META OBJECT_LOCATION OBJECT_BODY", ctx->object_key);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
@@ -259,8 +259,8 @@ int tango_cache_try_fetch_redis(struct tango_cache_ctx *ctx)
{
int ret = -1;
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx, "HMGET %s/%s OBJECT_META OBJECT_LOCATION OBJECT_BODY",
ctx->instance->param->bucketname, ctx->object_key);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx,
"HMGET %s OBJECT_META OBJECT_LOCATION OBJECT_BODY", ctx->object_key);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
@@ -295,8 +295,8 @@ static void redis_hset_command_cb(struct redisClusterAsyncContext *ac, void *vre
{
case PUT_STATE_REDIS_META:
case PUT_STATE_REDIS_ALL:
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx, "EXPIRE %s/%s %u",
ctx->instance->param->bucketname, ctx->object_key, ctx->put.object_ttl);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx,
"EXPIRE %s %u", ctx->object_key, ctx->put.object_ttl);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_EXEC);
@@ -321,8 +321,8 @@ int redis_put_minio_object_meta(struct tango_cache_ctx *ctx, bool callback)
char *meta;
meta = cJSON_PrintUnformatted(ctx->put.object_meta);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx, "HMSET %s/%s OBJECT_LOCATION minio OBJECT_META %s",
ctx->instance->param->bucketname, ctx->object_key, meta);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx,
"HMSET %s OBJECT_LOCATION minio OBJECT_META %s", ctx->object_key, meta);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
@@ -344,8 +344,8 @@ int redis_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_CO
ctx->instance->statistic.memory_used -= size;
meta = cJSON_PrintUnformatted(ctx->put.object_meta);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx, "HMSET %s/%s OBJECT_LOCATION redis OBJECT_META %s OBJECT_BODY %b",
ctx->instance->param->bucketname, ctx->object_key, meta, data, size);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx,
"HMSET %s OBJECT_LOCATION redis OBJECT_META %s OBJECT_BODY %b", ctx->object_key, meta, data, size);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);

16
cache/src/tango_cache_transfer.cpp vendored
View File

@@ -124,11 +124,11 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
ctx->put.upload_offset = 0;
if(full)
{
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key);
snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key);
}
else
{
snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key, ++ctx->put.part_index, ctx->put.uploadID);
snprintf(minio_url, 256, "http://%s/%s?partNumber=%d&uploadId=%s", ctx->hostaddr, ctx->object_key, ++ctx->put.part_index, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
}
@@ -185,7 +185,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/%s?uploads", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key);
snprintf(minio_url, 256, "http://%s/%s?uploads", ctx->hostaddr, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, 0); //Ĭ<><C4AC>ʹ<EFBFBD>ûص<C3BB><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>fread<61><64><EFBFBD><EFBFBD><EFBFBD>Է<EFBFBD><D4B7>ֹر<D6B9>Expectʱ<74>ᵼ<EFBFBD>¿<EFBFBD><C2BF><EFBFBD>curl_multi_socket_action
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
@@ -215,7 +215,7 @@ int cache_delete_minio_object(struct tango_cache_ctx *ctx, bool call_back)
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key);
snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
@@ -239,7 +239,7 @@ bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx)
return false;
}
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key, ctx->put.uploadID);
snprintf(minio_url, 256, "http://%s/%s?uploadId=%s", ctx->hostaddr, ctx->object_key, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
@@ -265,7 +265,7 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
}
construct_complete_xml(ctx, &ctx->put.combine_xml, &len);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key, ctx->put.uploadID);
snprintf(minio_url, 256, "http://%s/%s?uploadId=%s", ctx->hostaddr, ctx->object_key, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
@@ -522,7 +522,7 @@ int http_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COP
}
ctx->put.state = PUT_STATE_END;
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key);
snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
@@ -901,7 +901,7 @@ static int tango_cache_fetch_minio(struct tango_cache_ctx *ctx)
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->hostaddr, ctx->instance->param->bucketname, ctx->object_key);
snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
if(ctx->method == CACHE_REQUEST_HEAD)
{