gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改FETCH API函数名为tango_cache_fetch_object；增加DELETE API；调整ctx结构体内部结构；

Browse Source
This commit is contained in:
zhangchengwei
2018-09-27 15:04:56 +08:00
committed by zhengchao
parent dc6a6331d3
commit ce3e46b6cf
11 changed files with 364 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
cache/cache_evbase_client.cpp vendored
View File

@@ -26,6 +26,7 @@ enum CACHE_ASYN_CMD
CACHE_ASYN_UPLOAD_FRAG_DATA,
CACHE_ASYN_UPLOAD_FRAG_EVBUF,
CACHE_ASYN_UPLOAD_END,
CACHE_ASYN_DELETE,
};
struct databuffer
@@ -157,6 +158,11 @@ static void cache_asyn_ioevent_dispatch(struct databuffer *buffer)
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_DELETE:
cache_delete_minio_object(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_UPLOAD_ONCE_DATA:
tango_cache_upload_once_start_data(ctx_asyn->ctx, PUT_MEM_FREE, buffer->data, buffer->size);
cache_asyn_ctx_destroy(ctx_asyn);
@@ -234,23 +240,19 @@ void cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn)
{
struct databuffer *buffer;
if(ctx_asyn->ctx->fail_state)
{
tango_cache_ctx_destroy(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn);
return;
}
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_END;
if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{
ctx_asyn->ctx->fail_state = true;
if(ctx_asyn->ctx->future != NULL)
if(!ctx_asyn->ctx->fail_state)
{
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error");
ctx_asyn->ctx->fail_state = true;
if(ctx_asyn->ctx->future != NULL)
{
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error");
}
}
tango_cache_ctx_destroy(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn);
@@ -292,7 +294,6 @@ int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_ME
free(buffer);
return -1;
}
return 0;
}
@@ -379,11 +380,6 @@ int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{
ctx_asyn->ctx->fail_state = true;
if(ctx_asyn->ctx->future != NULL)
{
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error");
}
free(buffer->data);
free(buffer);
tango_cache_ctx_destroy(ctx);
@@ -422,11 +418,6 @@ int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struc
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{
ctx_asyn->ctx->fail_state = true;
if(ctx_asyn->ctx->future != NULL)
{
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error");
}
evbuffer_free(buffer->evbuf);
free(buffer);
tango_cache_ctx_destroy(ctx);
@@ -436,7 +427,7 @@ int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struc
return 0;
}
int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta)
int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta)
{
struct cache_evbase_ctx *ctx_asyn;
struct databuffer *buffer;
@@ -451,11 +442,29 @@ int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* fu
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{
ctx_asyn->ctx->fail_state = true;
if(ctx_asyn->ctx->future != NULL)
{
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error");
}
tango_cache_ctx_destroy(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn);
free(buffer);
return -1;
}
return 0;
}
int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* future, const char *objkey)
{
struct cache_evbase_ctx *ctx_asyn;
struct databuffer *buffer;
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = tango_cache_delete_prepare(instance->instance, future, objkey);
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_DELETE;
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{
tango_cache_ctx_destroy(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn);
free(buffer);

11
cache/include/cache_evbase_client.h vendored
View File

@@ -20,20 +20,23 @@ struct cache_evbase_ctx
struct cache_evbase_instance *instance_asyn;
};
/*API<50><49>ʹ<EFBFBD><CAB9>˵<EFBFBD><CBB5><EFBFBD>ο<EFBFBD>tango_cache_client.h*/
/*<EFBFBD><EFBFBD><EFBFBD><EFBFBD>API<EFBFBD>̲߳<EFBFBD><EFBFBD><EFBFBD>ȫ<EFBFBD><EFBFBD>API<EFBFBD><EFBFBD>ʹ<EFBFBD><EFBFBD>˵<EFBFBD><EFBFBD><EFBFBD>ο<EFBFBD>tango_cache_client.h*/
enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn);
enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance);
void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out);
/*<2A><><EFBFBD><EFBFBD>ʵ<EFBFBD><CAB5><EFBFBD><EFBFBD>ÿ<EFBFBD>߳<EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʹ<EFBFBD><CAB9>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>*/
struct cache_evbase_instance *cache_evbase_instance_new(const char* profile_path, const char* section, void *runtimelog);
//GET<45>ӿ<EFBFBD>
int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta);
//GET<45>ӿڣ<EFBFBD><EFBFBD>ɹ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>0<EFBFBD><EFBFBD>ʧ<EFBFBD>ܷ<EFBFBD><EFBFBD><EFBFBD>-1<><31>future<72>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>߳<EFBFBD><DFB3><EFBFBD>ִ<EFBFBD>У<EFBFBD><D0A3><EFBFBD>ͬ
int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta);
struct tango_cache_result *cache_evbase_read_result(void *promise_result);
//DELETE<54>ӿ<EFBFBD>
int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* future, const char *objkey);
//һ<><D2BB><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD>
int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* future,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size,

19
cache/include/tango_cache_client.h vendored
View File

@@ -39,6 +39,9 @@ struct cache_statistics
long long put_recv_num; //<2F><><EFBFBD><EFBFBD>UPLOAD<41>Ĵ<EFBFBD><C4B4><EFBFBD>
long long put_succ_num; //UPLOAD<41>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long put_error_num;//UPLOADʧ<44>ܵĴ<DCB5><C4B4><EFBFBD>
long long del_recv_num; //<2F><><EFBFBD><EFBFBD>DELETE<54>Ĵ<EFBFBD><C4B4><EFBFBD>
long long del_succ_num; //DELETE<54>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long del_error_num;//DELETE<54>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long memory_used; //<2F><>ǰUPLOAD BODY<44><59>ռ<EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD>С
long long session_num; //<2F><>ǰ<EFBFBD><C7B0><EFBFBD>ڽ<EFBFBD><DABD><EFBFBD>GET/PUT<55><54>HTTP<54>Ự<EFBFBD><E1BBB0>
};
@@ -47,13 +50,13 @@ enum CACHE_RESULT_TYPE
{
RESULT_TYPE_HEADER=0, //ֻ<><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>
RESULT_TYPE_USERTAG, //ֻ<><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>
RESULT_TYPE_BODY,
RESULT_TYPE_BODY, //<2F><><EFBFBD>ܵ<EFBFBD><DCB5>ö<EFBFBD><C3B6><EFBFBD>
};
//promise_success<73>Ľ<EFBFBD><C4BD><EFBFBD>result
struct tango_cache_result
{
const void *data_frag; //<2F><><EFBFBD><EFBFBD>typeΪRESULT_TYPE_HEADER<45><52><EFBFBD><EFBFBD><EFBFBD>ݻ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
const void *data_frag; //<2F><><EFBFBD><EFBFBD>typeΪRESULT_TYPE_HEADER<45><52>ÿ<EFBFBD><EFBFBD>ͷ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>(HTTP1.1<EFBFBD><EFBFBD>ʽ)
size_t size;
enum CACHE_RESULT_TYPE type;
};
@@ -70,7 +73,7 @@ enum CACHE_HTTP_HDR_TYPE
struct tango_cache_meta
{
const char* url; //<2F><><EFBFBD><EFBFBD>:URL<52><4C><EFBFBD>ǽṹ<C7BD><E1B9B9><EFBFBD><EFBFBD>־:<3A>ļ<EFBFBD><C4BC><EFBFBD>
const char* url; //<2F><><EFBFBD><EFBFBD>:URL<52><4C><EFBFBD>ǽṹ<C7BD><E1B9B9><EFBFBD><EFBFBD>־:<3A>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>󳤶<EFBFBD>256<EFBFBD>ֽ<EFBFBD>
const char* std_hdr[HDR_CONTENT_NUM]; //<2F><><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"Content-Type: text/html"<22><><EFBFBD><EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>У<EFBFBD>NULL<4C><4C>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD>
const char* usertag; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD>GETʱ<54><CAB1>ԭ<EFBFBD><D4AD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
size_t usertag_len; //<2F><><EFBFBD>󳤶<EFBFBD>USER_TAG_MAX_LEN<45><4E>0<EFBFBD><30>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7>
@@ -92,16 +95,18 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str
struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,const char* profile_path, const char* section, void *runtimelog);
/* GET<45>ӿڵ<D3BF>API*/
//<2F>ɹ<EFBFBD>ʱ<EFBFBD>ص<EFBFBD>promise_success, resultΪNULLʱ<4C><CAB1>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
//ʧ<><CAA7>ʱ<EFBFBD>ص<EFBFBD>promise_failed(<28><>һ<EFBFBD><D2BB>)<29><>ʹ<EFBFBD><CAB9>get_last_error<6F><72>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD>룻
//future<72><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ΪNULL
int tango_cache_fetch(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
//<2F><>promise_success<73><73>result<6C><74><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ҪCaller<65><72><EFBFBD><EFBFBD>tango_cache_result_release<73>ͷţ<CDB7>
int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
//<2F><>promise_success<73><73>result<6C><74><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD>
struct tango_cache_result *tango_cache_read_result(void *promise_result);
/* DELETE<54>ӿڵ<D3BF>API*/
int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* future, const char *objkey);
/* UPLOAD<41>ӿڵ<D3BF>API
* ע<><D7A2>: UPLOAD<41>ӿڵ<D3BF>API<50><49><EFBFBD><EFBFBD>future<72><65>ΪNULL<4C><4C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>֪ͨ<CDA8>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>򲻵<EFBFBD><F2B2BBB5>ã<EFBFBD>

191
cache/tango_cache_client.cpp vendored
View File

@@ -16,7 +16,7 @@
#include "tango_cache_transfer.h"
#include "tango_cache_tools.h"
int TANGO_CACHE_VERSION_20180910=0;
int TANGO_CACHE_VERSION_20180925=0;
void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_t size)
{
@@ -51,6 +51,9 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str
out->put_recv_num = instance->statistic.put_recv_num;
out->put_succ_num = instance->statistic.put_succ_num;
out->put_error_num= instance->statistic.put_error_num;
out->del_recv_num = instance->statistic.del_recv_num;
out->del_succ_num = instance->statistic.del_succ_num;
out->del_error_num= instance->statistic.del_error_num;
out->session_num = instance->statistic.session_num;
out->memory_used = instance->statistic.memory_used;
}
@@ -67,30 +70,42 @@ const char *tango_cache_get_object_key(struct tango_cache_ctx *ctx)
static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, enum CACHE_ERR_CODE error_code, struct cache_statistics *statistic)
{
if(method == CACHE_REQUEST_PUT)
switch(method)
{
if(fail_state)
{
statistic->put_error_num += 1;
}
else
{
statistic->put_succ_num += 1;
}
}
else
{
if(fail_state)
{
if(error_code == CACHE_ERR_CURL)
statistic->get_error_num += 1;
case CACHE_REQUEST_PUT:
if(fail_state)
{
statistic->put_error_num += 1;
}
else
statistic->get_miss_num += 1;
}
else
{
statistic->get_succ_num += 1;
}
{
statistic->put_succ_num += 1;
}
break;
case CACHE_REQUEST_GET:
if(fail_state)
{
if(error_code == CACHE_ERR_CURL)
statistic->get_error_num += 1;
else
statistic->get_miss_num += 1;
}
else
{
statistic->get_succ_num += 1;
}
break;
case CACHE_REQUEST_DELETE:
if(fail_state)
{
statistic->del_error_num += 1;
}
else
{
statistic->del_succ_num += 1;
}
break;
default:break;
}
}
@@ -119,6 +134,8 @@ void easy_string_savedata(struct easy_string *estr, const char *data, size_t len
void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx)
{
struct multipart_etag_list *etag;
if(ctx->curl != NULL)
{
curl_multi_remove_handle(ctx->instance->multi_hd, ctx->curl);
@@ -126,37 +143,37 @@ void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx)
}
easy_string_destroy(&ctx->response);
if(ctx->method == CACHE_REQUEST_GET)
switch(ctx->method)
{
easy_string_destroy(&ctx->response_tag);
}
else
{
struct multipart_etag_list *etag;
case CACHE_REQUEST_GET:
easy_string_destroy(&ctx->get.response_tag);
break;
if(ctx->uploadID != NULL) free(ctx->uploadID);
if(ctx->combine_xml != NULL) free(ctx->combine_xml);
if(ctx->headers != NULL) curl_slist_free_all(ctx->headers);
if(ctx->evbuffer!=NULL) evbuffer_free(ctx->evbuffer);
TAILQ_FOREACH(etag, &ctx->cache_head, node)
{
TAILQ_REMOVE(&ctx->cache_head, etag, node);
free(etag->etag);
free(etag);
}
if(ctx->future != NULL)
{
if(ctx->fail_state)
case CACHE_REQUEST_PUT:
if(ctx->put.uploadID != NULL) free(ctx->put.uploadID);
if(ctx->put.combine_xml != NULL) free(ctx->put.combine_xml);
if(ctx->headers != NULL) curl_slist_free_all(ctx->headers);
if(ctx->put.evbuf!=NULL) evbuffer_free(ctx->put.evbuf);
TAILQ_FOREACH(etag, &ctx->put.etag_head, node)
{
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, ctx->error);
}
else
TAILQ_REMOVE(&ctx->put.etag_head, etag, node);
free(etag->etag);
free(etag);
}//no break here
case CACHE_REQUEST_DELETE:
if(ctx->future != NULL)
{
promise_success(future_to_promise(ctx->future), NULL);
if(ctx->fail_state)
{
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, ctx->error);
}
else
{
promise_success(future_to_promise(ctx->future), NULL);
}
}
}
break;
default: break;
}
update_statistics(ctx->method, ctx->fail_state, ctx->error_code, &ctx->instance->statistic);
free(ctx);
@@ -173,12 +190,12 @@ int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data,
{
return -1;
}
if(evbuffer_add(ctx->evbuffer, data, size))
if(evbuffer_add(ctx->put.evbuf, data, size))
{
return -1;
}
ctx->instance->statistic.memory_used += size;
if(evbuffer_get_length(ctx->evbuffer) >= ctx->instance->upload_block_size)
if(evbuffer_get_length(ctx->put.evbuf) >= ctx->instance->upload_block_size)
{
cache_kick_upload_minio_multipart(ctx, ctx->instance->upload_block_size);
}
@@ -197,20 +214,20 @@ int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COP
size = evbuffer_get_length(evbuf);
if(way == EVBUFFER_MOVE)
{
if(evbuffer_add_buffer(ctx->evbuffer, evbuf))
if(evbuffer_add_buffer(ctx->put.evbuf, evbuf))
{
return -1;
}
}
else
{
if(evbuffer_add_buffer_reference(ctx->evbuffer, evbuf))
if(evbuffer_add_buffer_reference(ctx->put.evbuf, evbuf))
{
return -1;
}
}
ctx->instance->statistic.memory_used += size;
if(evbuffer_get_length(ctx->evbuffer) >= ctx->instance->upload_block_size)
if(evbuffer_get_length(ctx->put.evbuf) >= ctx->instance->upload_block_size)
{
cache_kick_upload_minio_multipart(ctx, ctx->instance->upload_block_size);
}
@@ -296,8 +313,8 @@ struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *in
return NULL;
}
ctx->evbuffer = evbuffer_new();
TAILQ_INIT(&ctx->cache_head);
ctx->put.evbuf = evbuffer_new();
TAILQ_INIT(&ctx->put.etag_head);
return ctx;
}
@@ -346,9 +363,9 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i
ctx->instance = instance;
ctx->future = future;
ctx->method = CACHE_REQUEST_GET;
ctx->get_state = GET_STATE_START;
ctx->max_age = meta->get.max_age;
ctx->min_fresh = meta->get.min_fresh;
ctx->get.state = GET_STATE_START;
ctx->get.max_age = meta->get.max_age;
ctx->get.min_fresh = meta->get.min_fresh;
instance->statistic.get_recv_num += 1;
if(instance->hash_object_key)
@@ -360,15 +377,42 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i
{
snprintf(ctx->object_key, 256, "%s", meta->url);
}
return ctx;
}
int tango_cache_fetch(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta)
int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta)
{
return tango_cache_fetch_start(tango_cache_fetch_prepare(instance, future, meta));
}
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* future, const char *objkey)
{
struct tango_cache_ctx *ctx;
char sha256[72]={0};
ctx = (struct tango_cache_ctx *)calloc(1, sizeof(struct tango_cache_ctx));
ctx->instance = instance;
ctx->future = future;
ctx->method = CACHE_REQUEST_DELETE;
instance->statistic.del_recv_num += 1;
if(instance->hash_object_key)
{
caculate_sha256(objkey, strlen(objkey), sha256, 72);
snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
}
else
{
snprintf(ctx->object_key, 256, "%s", objkey);
}
return ctx;
}
int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* future, const char *objkey)
{
return (cache_delete_minio_object(tango_cache_delete_prepare(instance, future, objkey))==1)?0:-1;
}
static void check_multi_info(CURLM *multi)
{
CURLMsg *msg;
@@ -394,13 +438,18 @@ static void check_multi_info(CURLM *multi)
ctx->curl = NULL;
ctx->res_code = 0;
if(ctx->method == CACHE_REQUEST_GET)
switch(ctx->method)
{
tango_cache_curl_get_done(easy, ctx, res, res_code);
}
else
{
tango_cache_curl_put_done(easy, ctx, res, res_code);
case CACHE_REQUEST_GET:
tango_cache_curl_get_done(ctx, res, res_code);
break;
case CACHE_REQUEST_PUT:
tango_cache_curl_put_done(ctx, res, res_code);
break;
case CACHE_REQUEST_DELETE:
tango_cache_curl_del_done(ctx, res, res_code);
break;
default: break;
}
}
}
@@ -519,19 +568,13 @@ static int load_local_configure(struct tango_cache_instance *instance, const cha
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] MINIO_BROKERS_LIST not found.\n", profile_path, section);
return -1;
}
MESA_load_profile_uint_def(profile_path, section, "CACHE_BLOCK_MAX_SIZE", &instance->block_len, 8192);
if(instance->block_len > 16777216)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_BLOCK_MAX_SIZE too large, must smaller than 16777216(16MB).\n", profile_path, section);
return -1;
}
MESA_load_profile_uint_def(profile_path, section, "CACHE_UPLOAD_BLOCK_SIZE", &instance->upload_block_size, 5242880);
if(instance->upload_block_size < 5242880)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_UPLOAD_BLOCK_SIZE too small, must bigger than 5242880(5MB).\n", profile_path, section);
return -1;
}
MESA_load_profile_uint_def(profile_path, section, "CACHE_DEFAULT_TTL_SECOND", &intval, 3600);
MESA_load_profile_uint_def(profile_path, section, "CACHE_DEFAULT_TTL_SECOND", &intval, 999999999);
if(intval < 60)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_DEFAULT_TTL_SECOND too small, must bigger than 60s.\n", profile_path, section);

58
cache/tango_cache_client_in.h vendored
View File

@@ -17,6 +17,7 @@ enum CACHE_REQUEST_METHOD
{
CACHE_REQUEST_GET=0,
CACHE_REQUEST_PUT,
CACHE_REQUEST_DELETE,
};
enum GET_OBJECT_STATE
@@ -54,7 +55,6 @@ struct tango_cache_instance
time_t relative_ttl; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ч<EFBFBD><D0A7>
u_int64_t cache_limit_size;
long max_cnn_host;
u_int32_t block_len; //<2F><><EFBFBD><EFBFBD>buffercache<68>ڴ<EFBFBD><DAB4>Ļ<EFBFBD><C4BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1>ÿ<EFBFBD><C3BF>update<74><65><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD>ò<EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ӿ<EFBFBD><D3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
u_int32_t upload_block_size; //minio<69>ֶ<EFBFBD><D6B6>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD><EFBFBD>
enum CACHE_ERR_CODE error_code;
u_int32_t hash_object_key;
@@ -67,6 +67,30 @@ struct multipart_etag_list
TAILQ_ENTRY(multipart_etag_list) node;
};
struct cache_ctx_data_get
{
time_t max_age;//Get
time_t min_fresh;//Get
time_t expires;
time_t last_modify;
u_int32_t need_hdrs;
enum GET_OBJECT_STATE state;
struct easy_string response_tag;
};
struct cache_ctx_data_put
{
struct evbuffer *evbuf;
size_t upload_length;
size_t upload_offset;
char *uploadID;
char *combine_xml;
TAILQ_HEAD(__etag_list_head, multipart_etag_list) etag_head;
enum PUT_OBJECT_STATE state;
u_int32_t part_index; //<2F><>RESPONSE_HDR_
bool close_state; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ùر<C3B9>
};
struct tango_cache_ctx
{
CURL *curl;
@@ -74,34 +98,18 @@ struct tango_cache_ctx
struct future* future;
char error[CURL_ERROR_SIZE];
char object_key[256];
char hostport[24]; //<2F><>ͬctxʹ<78><CAB9><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC>IP<49><50><EFBFBD><EFBFBD>֤pipeline˳<65><CBB3><EFBFBD><EFBFBD>
enum CACHE_REQUEST_METHOD method;
enum CACHE_ERR_CODE error_code;
struct evbuffer *evbuffer;
union{
enum PUT_OBJECT_STATE put_state;
enum GET_OBJECT_STATE get_state;
};
u_int32_t part_index;
u_int32_t need_hdrs; //<2F><>RESPONSE_HDR_
bool fail_state;
bool close_state; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ùر<C3B9>
long res_code;
time_t max_age;//Get
time_t min_fresh;//Get
time_t expires;
time_t last_modify;
size_t upload_length;
size_t upload_offset;
char *uploadID;
char *combine_xml;
struct easy_string response;
struct easy_string response_tag;
TAILQ_HEAD(__etag_list_head, multipart_etag_list) cache_head;
bool fail_state;
long res_code;
union{
struct cache_ctx_data_put put;
struct cache_ctx_data_get get;
};
struct tango_cache_instance *instance;
};
@@ -111,13 +119,13 @@ struct curl_socket_data
};
void easy_string_savedata(struct easy_string *estr, const char *data, size_t len);
void easy_string_expand(struct easy_string *estr, size_t to_size);
void easy_string_destroy(struct easy_string *estr);
void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx);
struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* future, const char *objkey);
#endif

208
cache/tango_cache_transfer.cpp vendored
View File

@@ -36,10 +36,10 @@ static size_t curl_put_multipart_header_cb(void *ptr, size_t size, size_t count,
etag = (struct multipart_etag_list *)malloc(sizeof(struct multipart_etag_list));
totallen = end - start + 1;
etag->etag = (char *)malloc(totallen + 1);
etag->part_number = ctx->part_index;
etag->part_number = ctx->put.part_index;
memcpy(etag->etag, start, totallen);
*(etag->etag + totallen) = '\0';
TAILQ_INSERT_TAIL(&ctx->cache_head, etag, node);
TAILQ_INSERT_TAIL(&ctx->put.etag_head, etag, node);
}
}
@@ -78,19 +78,19 @@ static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, v
size_t len, space=size*count, send_len;
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
if(size==0 || count==0 || ctx->upload_offset>=ctx->upload_length)
if(size==0 || count==0 || ctx->put.upload_offset>=ctx->put.upload_length)
{
return 0;
}
len = ctx->upload_length - ctx->upload_offset;
len = ctx->put.upload_length - ctx->put.upload_offset;
if(len > space)
{
len = space;
}
send_len = evbuffer_remove(ctx->evbuffer, ptr, len);
send_len = evbuffer_remove(ctx->put.evbuf, ptr, len);
assert(send_len>0);
ctx->upload_offset += send_len;
ctx->put.upload_offset += send_len;
ctx->instance->statistic.memory_used -= send_len;
return send_len;
@@ -107,14 +107,14 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
return -1;
}
ctx->upload_offset = 0;
ctx->put.upload_offset = 0;
if(full)
{
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
}
else
{
snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ++ctx->part_index, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ++ctx->put.part_index, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
}
@@ -132,7 +132,7 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->upload_length);
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put.upload_length);
curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb);
curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);
@@ -199,14 +199,14 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
return 1;
}
bool cache_delete_minio_object(struct tango_cache_ctx *ctx)
int cache_delete_minio_object(struct tango_cache_ctx *ctx)
{
CURLMcode rc;
char minio_url[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
return false;
return 0;
}
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
@@ -223,7 +223,7 @@ bool cache_delete_minio_object(struct tango_cache_ctx *ctx)
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
return true;
return 1;
}
//return value: true-<2D>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD><C2BC><EFBFBD>false-δ<><CEB4><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD>
@@ -237,7 +237,7 @@ bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx)
return false;
}
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -265,9 +265,9 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
{
return false;
}
construct_complete_xml(ctx, &ctx->combine_xml, &len);
construct_complete_xml(ctx, &ctx->put.combine_xml, &len);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -279,7 +279,7 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->combine_xml);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->put.combine_xml);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, len); //<2F><><EFBFBD><EFBFBD>Content-Length
if(ctx->headers != NULL)
@@ -300,17 +300,17 @@ bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block
{
int ret = 1;
switch(ctx->put_state)
switch(ctx->put.state)
{
case PUT_STATE_START:
ctx->put_state = PUT_STATE_WAIT_START;
ctx->put.state = PUT_STATE_WAIT_START;
ret = curl_get_minio_uploadID(ctx);
break;
case PUT_STATE_PART:
if(ctx->curl == NULL)
{
ctx->upload_length = block_len;
ctx->put.upload_length = block_len;
ret = http_put_bodypart_request_evbuf(ctx, false);
}
break;
@@ -331,9 +331,9 @@ int http_put_complete_part_evbuf(struct tango_cache_ctx *ctx)
{
int ret=0;
ctx->put_state = PUT_STATE_END;
ctx->upload_length = evbuffer_get_length(ctx->evbuffer);
if(ctx->upload_length > 0)
ctx->put.state = PUT_STATE_END;
ctx->put.upload_length = evbuffer_get_length(ctx->put.evbuf);
if(ctx->put.upload_length > 0)
{
ret = http_put_bodypart_request_evbuf(ctx, true);
if(ret <= 0)
@@ -355,14 +355,14 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
{
int ret = 0;
ctx->close_state = true;//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>رգ<D8B1><D5A3>ڲ<EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD>ת<EFBFBD><D7AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ٹر<D9B9>
ctx->put.close_state = true;//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>رգ<D8B1><D5A3>ڲ<EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD>ת<EFBFBD><D7AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ٹر<D9B9>
if(ctx->fail_state)
{
tango_cache_ctx_destroy(ctx);
return 0;
}
switch(ctx->put_state)
switch(ctx->put.state)
{
case PUT_STATE_START:
http_put_complete_part_evbuf(ctx);
@@ -371,12 +371,12 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
case PUT_STATE_PART:
if(ctx->curl == NULL)
{
ctx->upload_length = evbuffer_get_length(ctx->evbuffer);
if(ctx->upload_length == 0)
ctx->put.upload_length = evbuffer_get_length(ctx->put.evbuf);
if(ctx->put.upload_length == 0)
{
if(cache_kick_combine_minio(ctx))
{
ctx->put_state = PUT_STATE_END;
ctx->put.state = PUT_STATE_END;
}
else
{
@@ -394,7 +394,7 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
ctx->error_code = CACHE_ERR_CURL;
if(cache_cancel_upload_minio(ctx))
{
ctx->put_state = PUT_STATE_CANCEL;
ctx->put.state = PUT_STATE_CANCEL;
}
else
{
@@ -413,18 +413,18 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
return ret;
}
void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code)
void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
switch(ctx->put_state)
switch(ctx->put.state)
{
case PUT_STATE_WAIT_START:
if(res!=CURLE_OK||res_code!=200L|| ctx->fail_state || !parse_uploadID_xml(ctx->response.buff, ctx->response.len, &ctx->uploadID))
if(res!=CURLE_OK||res_code!=200L|| ctx->fail_state || !parse_uploadID_xml(ctx->response.buff, ctx->response.len, &ctx->put.uploadID))
{
easy_string_destroy(&ctx->response);
ctx->error_code = CACHE_ERR_CURL;
ctx->fail_state = true;
if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
if(ctx->close_state)
if(ctx->put.close_state)
{
tango_cache_ctx_destroy(ctx);
}
@@ -432,14 +432,14 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
else
{
easy_string_destroy(&ctx->response);
ctx->put_state = PUT_STATE_PART;
if(ctx->close_state)
ctx->put.state = PUT_STATE_PART;
if(ctx->put.close_state)
{
cache_kick_upload_minio_end(ctx);
}
else
{
size_t upload_length = evbuffer_get_length(ctx->evbuffer);
size_t upload_length = evbuffer_get_length(ctx->put.evbuf);
if(upload_length >= ctx->instance->upload_block_size)
{
cache_kick_upload_minio_multipart(ctx, upload_length);
@@ -459,20 +459,20 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
{
if(cache_cancel_upload_minio(ctx))
{
ctx->put_state = PUT_STATE_CANCEL;
ctx->put.state = PUT_STATE_CANCEL;
}
else if(ctx->close_state)
else if(ctx->put.close_state)
{
tango_cache_ctx_destroy(ctx);
}
}
else if(ctx->close_state)
else if(ctx->put.close_state)
{
cache_kick_upload_minio_end(ctx);
}
else
{
size_t upload_length = evbuffer_get_length(ctx->evbuffer);
size_t upload_length = evbuffer_get_length(ctx->put.evbuf);
if(upload_length >= ctx->instance->upload_block_size)
{
cache_kick_upload_minio_multipart(ctx, upload_length);
@@ -481,7 +481,7 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
break;
case PUT_STATE_CANCEL: //<2F>ȴ<EFBFBD><C8B4>ر<EFBFBD>
if(ctx->close_state)
if(ctx->put.close_state)
{
tango_cache_ctx_destroy(ctx);
}
@@ -514,7 +514,7 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM
}
return -1;
}
ctx->put_state = PUT_STATE_END;
ctx->put.state = PUT_STATE_END;
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
@@ -556,18 +556,18 @@ int tango_cache_upload_once_start_evbuf(struct tango_cache_ctx *ctx, enum EVBUFF
{
size_t size;
ctx->evbuffer = evbuffer_new();
ctx->put.evbuf = evbuffer_new();
size = evbuffer_get_length(evbuf);
if(way == EVBUFFER_MOVE)
{
if(evbuffer_add_buffer(ctx->evbuffer, evbuf))
if(evbuffer_add_buffer(ctx->put.evbuf, evbuf))
{
return -1;
}
}
else
{
if(evbuffer_add_buffer_reference(ctx->evbuffer, evbuf))
if(evbuffer_add_buffer_reference(ctx->put.evbuf, evbuf))
{
return -1;
}
@@ -577,9 +577,18 @@ int tango_cache_upload_once_start_evbuf(struct tango_cache_ctx *ctx, enum EVBUFF
return http_put_complete_part_evbuf(ctx);
}
void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code)
void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
switch(ctx->get_state)
if(res!=CURLE_OK || (res_code!=204L && res_code!=200L ))
{
ctx->fail_state = true;
}
tango_cache_ctx_destroy(ctx);
}
void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
switch(ctx->get.state)
{
case GET_STATE_START:
if(!ctx->fail_state)
@@ -600,7 +609,7 @@ void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
case GET_STATE_DELETE:
if(cache_delete_minio_object(ctx))
{
ctx->get_state = GET_STATE_END;
ctx->get.state = GET_STATE_END;
}
else
{
@@ -620,27 +629,28 @@ static size_t curl_get_response_body_cb(void *ptr, size_t size, size_t count, vo
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
struct tango_cache_result result;
if(ctx->fail_state || ctx->get_state==GET_STATE_DELETE)
if(ctx->fail_state || ctx->get.state==GET_STATE_DELETE)
{
return size*count;
}
if(ctx->need_hdrs!=RESPONSE_HDR_ALL) //<2F><>Expiresʱ
if(ctx->get.need_hdrs!=RESPONSE_HDR_ALL) //<2F><>Expiresʱ
{
ctx->fail_state = true;
ctx->error_code = CACHE_ERR_CURL;
ctx->get_state = GET_STATE_DELETE;
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache Expires or last-modify not found");
ctx->get.state = GET_STATE_DELETE;
ctx->instance->statistic.del_recv_num += 1;
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache Expires or x-amz-meta-lm not found");
return size*count;
}
if(ctx->response_tag.len > 0)
if(ctx->get.response_tag.len > 0)
{
result.data_frag = ctx->response_tag.buff;
result.size = ctx->response_tag.len;
result.data_frag = ctx->get.response_tag.buff;
result.size = ctx->get.response_tag.len;
result.type = RESULT_TYPE_USERTAG;
promise_success(future_to_promise(ctx->future), &result);
easy_string_destroy(&ctx->response_tag);
easy_string_destroy(&ctx->get.response_tag);
}
if(ctx->response.len > 0)
{
@@ -661,14 +671,15 @@ static bool check_expires_header(struct tango_cache_ctx *ctx, const char *expire
{
time_t time_gmt;
ctx->expires = expires_hdr2timestamp(expires_val, len);
ctx->get.expires = expires_hdr2timestamp(expires_val, len);
time_gmt = get_gmtime_timestamp(time(NULL));
if(time_gmt > ctx->expires) //<2F><><EFBFBD><EFBFBD>ʧЧ<CAA7><D0A7>TODO relative_age<67>ĺ<EFBFBD><C4BA><EFBFBD><EFBFBD><EFBFBD>ɶ
if(time_gmt > ctx->get.expires) //<2F><><EFBFBD><EFBFBD>ʧЧ<CAA7><D0A7>TODO relative_age<67>ĺ<EFBFBD><C4BA><EFBFBD><EFBFBD><EFBFBD>ɶ
{
ctx->fail_state = true;
ctx->error_code = CACHE_TIMEOUT;
ctx->get_state = GET_STATE_DELETE; //<2F><><EFBFBD><EFBFBD>ʧЧʱ<D0A7><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
ctx->get.state = GET_STATE_DELETE; //<2F><><EFBFBD><EFBFBD>ʧЧʱ<D0A7><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
ctx->instance->statistic.del_recv_num += 1;
easy_string_destroy(&ctx->response);
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not fresh");
return false;
@@ -680,11 +691,11 @@ static bool check_fresh_header(struct tango_cache_ctx *ctx)
{
time_t now_gmt;
if(ctx->need_hdrs != RESPONSE_HDR_ALL)
if(ctx->get.need_hdrs != RESPONSE_HDR_ALL)
return true;
now_gmt = get_gmtime_timestamp(time(NULL));
if(ctx->last_modify+ctx->max_age > now_gmt || now_gmt+ctx->min_fresh>ctx->expires)
if(ctx->get.last_modify+ctx->get.max_age > now_gmt || now_gmt+ctx->get.min_fresh>ctx->get.expires)
{
ctx->fail_state = true;
ctx->error_code = CACHE_TIMEOUT;
@@ -716,8 +727,9 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
char *start=(char *)ptr, *pos_colon;
size_t raw_len = size*count, hdrlen=size*count;
char usertag[2048];
size_t datalen;
if(ctx->fail_state || ctx->get_state==GET_STATE_DELETE)
if(ctx->fail_state || ctx->get.state==GET_STATE_DELETE)
{
return raw_len;
}
@@ -725,49 +737,51 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
{
return raw_len;
}
if((pos_colon=(char*)memchr(start, ':', raw_len))!=NULL)
pos_colon = (char*)memchr(start, ':', raw_len);
if(pos_colon == NULL)
{
size_t datalen = pos_colon - start;
switch(datalen)
{
case 7:
if(strcmp_one_word_mesa_equal_len("expires", "EXPIRES", start, 7))
return raw_len;
}
datalen = pos_colon - start;
switch(datalen)
{
case 7:
if(strcmp_one_word_mesa_equal_len("expires", "EXPIRES", start, 7))
{
ctx->get.need_hdrs |= RESPONSE_HDR_EXPIRES;
if(!check_expires_header(ctx, pos_colon + 1, raw_len - datalen - 1) || !check_fresh_header(ctx))
{
ctx->need_hdrs |= RESPONSE_HDR_EXPIRES;
if(!check_expires_header(ctx, pos_colon + 1, raw_len - datalen - 1) || !check_fresh_header(ctx))
{
return raw_len;
}
return raw_len;
}
break;
case 13:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-lm", "X-AMZ-META-LM", start, 13))
}
break;
case 13:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-lm", "X-AMZ-META-LM", start, 13))
{
ctx->get.need_hdrs |= RESPONSE_HDR_LAST_MOD;
sscanf(pos_colon+1, "%lu", &ctx->get.last_modify);
if(!check_fresh_header(ctx))
{
ctx->need_hdrs |= RESPONSE_HDR_LAST_MOD;
sscanf(pos_colon+1, "%lu", &ctx->last_modify);
if(!check_fresh_header(ctx))
{
return raw_len;
}
return raw_len;
}
break;
case 15:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15))
}
break;
case 15:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15))
{
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0)
{
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0)
{
easy_string_savedata(&ctx->response_tag, usertag, hdrlen);
}
easy_string_savedata(&ctx->get.response_tag, usertag, hdrlen);
}
break;
case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 14: if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
default: break;
}
}
break;
case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 14: if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
default: break;
}
return raw_len;
}

7
cache/tango_cache_transfer.h vendored
View File

@@ -6,10 +6,11 @@
#include "tango_cache_client_in.h"
void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
bool cache_delete_minio_object(struct tango_cache_ctx *ctx);
int cache_delete_minio_object(struct tango_cache_ctx *ctx);
int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx);
bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block_len);

2
cache/tango_cache_xml.cpp vendored
View File

@@ -58,7 +58,7 @@ int construct_complete_xml(struct tango_cache_ctx *ctx, char **xml, int *len)
xmlNewProp(root, (const xmlChar *)"xmlns",(const xmlChar *)"http://s3.amazonaws.com/doc/2006-03-01/");
xmlDocSetRootElement(pdoc, root);
TAILQ_FOREACH(etag, &ctx->cache_head, node)
TAILQ_FOREACH(etag, &ctx->put.etag_head, node)
{
sprintf(number, "%u", etag->part_number);
child = xmlNewChild(root, NULL, (const xmlChar*)"Part", NULL);

51
cache/test_demo/cache_evbase_test.cpp vendored
View File

@@ -22,7 +22,7 @@
#include "cache_evbase_client.h"
struct cache_evbase_instance *instance_asyn;
int still_runing=1;
int runing_over=0;
struct future_pdata
{
@@ -59,15 +59,14 @@ void get_future_success(future_result_t* result, void * user)
future_destroy(pdata->future);
fclose(pdata->fp);
free(pdata);
still_runing = 0;
runing_over = 1;
}
}
void get_future_failed(enum e_future_error err, const char * what, void * user)
{
printf("GET fail: %s\n", what);
sleep(5);
still_runing = 0;
runing_over = 2;
}
void put_future_success(future_result_t* result, void * user)
@@ -77,7 +76,7 @@ void put_future_success(future_result_t* result, void * user)
printf("PUT %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
still_runing = 0;
runing_over = 1;
}
void put_future_failed(enum e_future_error err, const char * what, void * user)
{
@@ -86,7 +85,26 @@ void put_future_failed(enum e_future_error err, const char * what, void * user)
printf("PUT %s fail: %s\n", what, pdata->filename);
future_destroy(pdata->future);
free(pdata);
still_runing = 0;
runing_over = 1;
}
void del_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
void del_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s fail: %s\n", pdata->filename, what);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
char * get_file_content(const char *filename, size_t *filelen_out)
@@ -139,7 +157,7 @@ int main(int argc, char **argv)
if(argc != 2 && argc!=3)
{
printf("USGAE: %s <PUT/PUTONCE/PUTONCEEV/GET:filename> [get_out_file_index]\n", argv[0]);
printf("USGAE: %s <PUT/PUTONCE/PUTONCEEV/GET/DEL:filename> [get_out_file_index]\n", argv[0]);
return -1;
}
if(argc==3)
@@ -182,7 +200,14 @@ int main(int argc, char **argv)
promise_set_ctx(future_to_promise(pdata->future), NULL, NULL);
pdata->fp = fopen(filename_out, "w");
cache_evbase_fetch(instance_asyn, pdata->future, &meta);
cache_evbase_fetch_object(instance_asyn, pdata->future, &meta);
}
else if(!strcasecmp(p, "DEL"))
{
pdata->future = future_create(del_future_success, del_future_failed, pdata);
promise_set_ctx(future_to_promise(pdata->future), NULL, NULL);
sprintf(pdata->filename, "%s", filename_in);
cache_evbase_delete_object(instance_asyn, pdata->future, filename_in);
}
else if(!strcasecmp(p, "PUTONCE"))
{
@@ -237,7 +262,7 @@ int main(int argc, char **argv)
printf("Waiting to finish.......\n");
static int num=0;
while(still_runing)
while(!runing_over)
{
/*if(++num==10)
{
@@ -245,11 +270,15 @@ int main(int argc, char **argv)
}*/
sleep(1);
}
if(runing_over==2) //GETʱ<54><CAB1>ʱɾ<CAB1><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
{
sleep(5);
}
struct cache_statistics out;
cache_evbase_get_statistics(instance_asyn, &out);
printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.session_num, out.memory_used);
printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, del_recv: %llu, del_succ: %llu, del_fail: %llu, session: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.del_recv_num, out.del_succ_num, out.del_error_num, out.session_num, out.memory_used);
return 0;
}

3
cache/test_demo/pangu_tg_cahce.conf vendored
View File

@@ -8,9 +8,6 @@ MAX_CONNECTION_PER_HOST=10
#bucket的名称
CACHE_BUCKET_NAME=openbucket
#内部分配缓存块的大小，根据上传的文件大小分布而定，最大16777216（16MB）
CACHE_BLOCK_MAX_SIZE=8192
#缓存最大占用的内存空间大小，超出空间时上传失败
MAX_USED_MEMORY_SIZE_MB=5120

2
cache/test_demo/tango_cache_test.c vendored
View File

@@ -175,7 +175,7 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
pdata->future = future_create(get_future_success, get_future_failed, pdata);
promise_set_ctx(future_to_promise(pdata->future), NULL, NULL);
tango_cache_fetch(tango_instance, pdata->future, &meta);
tango_cache_fetch_object(tango_instance, pdata->future, &meta);
}
else if(!strcasecmp(p, "PUTONCE"))
{