gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改FETCH API函数名为tango_cache_fetch_object；增加DELETE API；调整ctx结构体内部结构；

Browse Source
This commit is contained in:
zhangchengwei
2018-09-27 15:04:56 +08:00
committed by zhengchao
parent dc6a6331d3
commit ce3e46b6cf
11 changed files with 364 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
cache/cache_evbase_client.cpp vendored
View File

@@ -26,6 +26,7 @@ enum CACHE_ASYN_CMD
CACHE_ASYN_UPLOAD_FRAG_DATA, CACHE_ASYN_UPLOAD_FRAG_DATA,
CACHE_ASYN_UPLOAD_FRAG_EVBUF, CACHE_ASYN_UPLOAD_FRAG_EVBUF,
CACHE_ASYN_UPLOAD_END, CACHE_ASYN_UPLOAD_END,
CACHE_ASYN_DELETE,
}; };
struct databuffer struct databuffer
@@ -157,6 +158,11 @@ static void cache_asyn_ioevent_dispatch(struct databuffer *buffer)
cache_asyn_ctx_destroy(ctx_asyn); cache_asyn_ctx_destroy(ctx_asyn);
break; break;
case CACHE_ASYN_DELETE:
cache_delete_minio_object(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_UPLOAD_ONCE_DATA: case CACHE_ASYN_UPLOAD_ONCE_DATA:
tango_cache_upload_once_start_data(ctx_asyn->ctx, PUT_MEM_FREE, buffer->data, buffer->size); tango_cache_upload_once_start_data(ctx_asyn->ctx, PUT_MEM_FREE, buffer->data, buffer->size);
cache_asyn_ctx_destroy(ctx_asyn); cache_asyn_ctx_destroy(ctx_asyn);
@@ -234,23 +240,19 @@ void cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn)
{ {
struct databuffer *buffer; struct databuffer *buffer;
if(ctx_asyn->ctx->fail_state)
{
tango_cache_ctx_destroy(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn);
return;
}
buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn; buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_END; buffer->cmd_type = CACHE_ASYN_UPLOAD_END;
if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *)) if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{ {
ctx_asyn->ctx->fail_state = true; if(!ctx_asyn->ctx->fail_state)
if(ctx_asyn->ctx->future != NULL)
{ {
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error"); ctx_asyn->ctx->fail_state = true;
if(ctx_asyn->ctx->future != NULL)
{
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error");
}
} }
tango_cache_ctx_destroy(ctx_asyn->ctx); tango_cache_ctx_destroy(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn); cache_asyn_ctx_destroy(ctx_asyn);
@@ -292,7 +294,6 @@ int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_ME
free(buffer); free(buffer);
return -1; return -1;
} }
return 0; return 0;
} }
@@ -379,11 +380,6 @@ int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *)) if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{ {
ctx_asyn->ctx->fail_state = true;
if(ctx_asyn->ctx->future != NULL)
{
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error");
}
free(buffer->data); free(buffer->data);
free(buffer); free(buffer);
tango_cache_ctx_destroy(ctx); tango_cache_ctx_destroy(ctx);
@@ -422,11 +418,6 @@ int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struc
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *)) if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{ {
ctx_asyn->ctx->fail_state = true;
if(ctx_asyn->ctx->future != NULL)
{
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error");
}
evbuffer_free(buffer->evbuf); evbuffer_free(buffer->evbuf);
free(buffer); free(buffer);
tango_cache_ctx_destroy(ctx); tango_cache_ctx_destroy(ctx);
@@ -436,7 +427,7 @@ int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struc
return 0; return 0;
} }
int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta) int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta)
{ {
struct cache_evbase_ctx *ctx_asyn; struct cache_evbase_ctx *ctx_asyn;
struct databuffer *buffer; struct databuffer *buffer;
@@ -451,11 +442,29 @@ int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* fu
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *)) if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{ {
ctx_asyn->ctx->fail_state = true; tango_cache_ctx_destroy(ctx_asyn->ctx);
if(ctx_asyn->ctx->future != NULL) cache_asyn_ctx_destroy(ctx_asyn);
{ free(buffer);
promise_failed(future_to_promise(ctx_asyn->ctx->future), FUTURE_ERROR_CANCEL, "write sockpair error"); return -1;
} }
return 0;
}
int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* future, const char *objkey)
{
struct cache_evbase_ctx *ctx_asyn;
struct databuffer *buffer;
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = tango_cache_delete_prepare(instance->instance, future, objkey);
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_DELETE;
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 0) != sizeof(void *))
{
tango_cache_ctx_destroy(ctx_asyn->ctx); tango_cache_ctx_destroy(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn); cache_asyn_ctx_destroy(ctx_asyn);
free(buffer); free(buffer);

11
cache/include/cache_evbase_client.h vendored
View File

@@ -20,20 +20,23 @@ struct cache_evbase_ctx
struct cache_evbase_instance *instance_asyn; struct cache_evbase_instance *instance_asyn;
}; };
/*API<50><49>ʹ<EFBFBD><CAB9>˵<EFBFBD><CBB5><EFBFBD>ο<EFBFBD>tango_cache_client.h*/ /*<EFBFBD><EFBFBD><EFBFBD><EFBFBD>API<EFBFBD>̲߳<EFBFBD><EFBFBD><EFBFBD>ȫ<EFBFBD><EFBFBD>API<EFBFBD><EFBFBD>ʹ<EFBFBD><EFBFBD>˵<EFBFBD><EFBFBD><EFBFBD>ο<EFBFBD>tango_cache_client.h*/
enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn); enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn);
enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance); enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance);
void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out); void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out);
/*<2A><><EFBFBD><EFBFBD>ʵ<EFBFBD><CAB5><EFBFBD><EFBFBD>ÿ<EFBFBD>߳<EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʹ<EFBFBD><CAB9>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>*/
struct cache_evbase_instance *cache_evbase_instance_new(const char* profile_path, const char* section, void *runtimelog); struct cache_evbase_instance *cache_evbase_instance_new(const char* profile_path, const char* section, void *runtimelog);
//GET<45>ӿ<EFBFBD> //GET<45>ӿڣ<EFBFBD><EFBFBD>ɹ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>0<EFBFBD><EFBFBD>ʧ<EFBFBD>ܷ<EFBFBD><EFBFBD><EFBFBD>-1<><31>future<72>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>߳<EFBFBD><DFB3><EFBFBD>ִ<EFBFBD>У<EFBFBD><D0A3><EFBFBD>ͬ
int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta); int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta);
struct tango_cache_result *cache_evbase_read_result(void *promise_result); struct tango_cache_result *cache_evbase_read_result(void *promise_result);
//DELETE<54>ӿ<EFBFBD>
int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* future, const char *objkey);
//һ<><D2BB><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD> //һ<><D2BB><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD>
int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* future, int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* future,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size,

19
cache/include/tango_cache_client.h vendored
View File

@@ -39,6 +39,9 @@ struct cache_statistics
long long put_recv_num; //<2F><><EFBFBD><EFBFBD>UPLOAD<41>Ĵ<EFBFBD><C4B4><EFBFBD> long long put_recv_num; //<2F><><EFBFBD><EFBFBD>UPLOAD<41>Ĵ<EFBFBD><C4B4><EFBFBD>
long long put_succ_num; //UPLOAD<41>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD> long long put_succ_num; //UPLOAD<41>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long put_error_num;//UPLOADʧ<44>ܵĴ<DCB5><C4B4><EFBFBD> long long put_error_num;//UPLOADʧ<44>ܵĴ<DCB5><C4B4><EFBFBD>
long long del_recv_num; //<2F><><EFBFBD><EFBFBD>DELETE<54>Ĵ<EFBFBD><C4B4><EFBFBD>
long long del_succ_num; //DELETE<54>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long del_error_num;//DELETE<54>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long memory_used; //<2F><>ǰUPLOAD BODY<44><59>ռ<EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD>С long long memory_used; //<2F><>ǰUPLOAD BODY<44><59>ռ<EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD>С
long long session_num; //<2F><>ǰ<EFBFBD><C7B0><EFBFBD>ڽ<EFBFBD><DABD><EFBFBD>GET/PUT<55><54>HTTP<54>Ự<EFBFBD><E1BBB0> long long session_num; //<2F><>ǰ<EFBFBD><C7B0><EFBFBD>ڽ<EFBFBD><DABD><EFBFBD>GET/PUT<55><54>HTTP<54>Ự<EFBFBD><E1BBB0>
}; };
@@ -47,13 +50,13 @@ enum CACHE_RESULT_TYPE
{ {
RESULT_TYPE_HEADER=0, //ֻ<><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB> RESULT_TYPE_HEADER=0, //ֻ<><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>
RESULT_TYPE_USERTAG, //ֻ<><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB> RESULT_TYPE_USERTAG, //ֻ<><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>
RESULT_TYPE_BODY, RESULT_TYPE_BODY, //<2F><><EFBFBD>ܵ<EFBFBD><DCB5>ö<EFBFBD><C3B6><EFBFBD>
}; };
//promise_success<73>Ľ<EFBFBD><C4BD><EFBFBD>result //promise_success<73>Ľ<EFBFBD><C4BD><EFBFBD>result
struct tango_cache_result struct tango_cache_result
{ {
const void *data_frag; //<2F><><EFBFBD><EFBFBD>typeΪRESULT_TYPE_HEADER<45><52><EFBFBD><EFBFBD><EFBFBD>ݻ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> const void *data_frag; //<2F><><EFBFBD><EFBFBD>typeΪRESULT_TYPE_HEADER<45><52>ÿ<EFBFBD><EFBFBD>ͷ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>(HTTP1.1<EFBFBD><EFBFBD>ʽ)
size_t size; size_t size;
enum CACHE_RESULT_TYPE type; enum CACHE_RESULT_TYPE type;
}; };
@@ -70,7 +73,7 @@ enum CACHE_HTTP_HDR_TYPE
struct tango_cache_meta struct tango_cache_meta
{ {
const char* url; //<2F><><EFBFBD><EFBFBD>:URL<52><4C><EFBFBD>ǽṹ<C7BD><E1B9B9><EFBFBD><EFBFBD>־:<3A>ļ<EFBFBD><C4BC><EFBFBD> const char* url; //<2F><><EFBFBD><EFBFBD>:URL<52><4C><EFBFBD>ǽṹ<C7BD><E1B9B9><EFBFBD><EFBFBD>־:<3A>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>󳤶<EFBFBD>256<EFBFBD>ֽ<EFBFBD>
const char* std_hdr[HDR_CONTENT_NUM]; //<2F><><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"Content-Type: text/html"<22><><EFBFBD><EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>У<EFBFBD>NULL<4C><4C>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD> const char* std_hdr[HDR_CONTENT_NUM]; //<2F><><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"Content-Type: text/html"<22><><EFBFBD><EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>У<EFBFBD>NULL<4C><4C>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD>
const char* usertag; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD>GETʱ<54><CAB1>ԭ<EFBFBD><D4AD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> const char* usertag; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD>GETʱ<54><CAB1>ԭ<EFBFBD><D4AD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
size_t usertag_len; //<2F><><EFBFBD>󳤶<EFBFBD>USER_TAG_MAX_LEN<45><4E>0<EFBFBD><30>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7> size_t usertag_len; //<2F><><EFBFBD>󳤶<EFBFBD>USER_TAG_MAX_LEN<45><4E>0<EFBFBD><30>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7>
@@ -92,16 +95,18 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str
struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,const char* profile_path, const char* section, void *runtimelog); struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,const char* profile_path, const char* section, void *runtimelog);
/* GET<45>ӿڵ<D3BF>API*/
//<2F>ɹ<EFBFBD>ʱ<EFBFBD>ص<EFBFBD>promise_success, resultΪNULLʱ<4C><CAB1>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD> //<2F>ɹ<EFBFBD>ʱ<EFBFBD>ص<EFBFBD>promise_success, resultΪNULLʱ<4C><CAB1>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
//ʧ<><CAA7>ʱ<EFBFBD>ص<EFBFBD>promise_failed(<28><>һ<EFBFBD><D2BB>)<29><>ʹ<EFBFBD><CAB9>get_last_error<6F><72>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD>룻 //ʧ<><CAA7>ʱ<EFBFBD>ص<EFBFBD>promise_failed(<28><>һ<EFBFBD><D2BB>)<29><>ʹ<EFBFBD><CAB9>get_last_error<6F><72>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD>룻
//future<72><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ΪNULL //future<72><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ΪNULL
int tango_cache_fetch(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta); int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
//<2F><>promise_success<73><73>result<6C><74><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD>
//<2F><>promise_success<73><73>result<6C><74><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ҪCaller<65><72><EFBFBD><EFBFBD>tango_cache_result_release<73>ͷţ<CDB7>
struct tango_cache_result *tango_cache_read_result(void *promise_result); struct tango_cache_result *tango_cache_read_result(void *promise_result);
/* DELETE<54>ӿڵ<D3BF>API*/
int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* future, const char *objkey);
/* UPLOAD<41>ӿڵ<D3BF>API /* UPLOAD<41>ӿڵ<D3BF>API
* ע<><D7A2>: UPLOAD<41>ӿڵ<D3BF>API<50><49><EFBFBD><EFBFBD>future<72><65>ΪNULL<4C><4C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>֪ͨ<CDA8>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>򲻵<EFBFBD><F2B2BBB5>ã<EFBFBD> * ע<><D7A2>: UPLOAD<41>ӿڵ<D3BF>API<50><49><EFBFBD><EFBFBD>future<72><65>ΪNULL<4C><4C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>֪ͨ<CDA8>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>򲻵<EFBFBD><F2B2BBB5>ã<EFBFBD>

191
cache/tango_cache_client.cpp vendored
View File

@@ -16,7 +16,7 @@
#include "tango_cache_transfer.h" #include "tango_cache_transfer.h"
#include "tango_cache_tools.h" #include "tango_cache_tools.h"
int TANGO_CACHE_VERSION_20180910=0; int TANGO_CACHE_VERSION_20180925=0;
void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_t size) void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_t size)
{ {
@@ -51,6 +51,9 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str
out->put_recv_num = instance->statistic.put_recv_num; out->put_recv_num = instance->statistic.put_recv_num;
out->put_succ_num = instance->statistic.put_succ_num; out->put_succ_num = instance->statistic.put_succ_num;
out->put_error_num= instance->statistic.put_error_num; out->put_error_num= instance->statistic.put_error_num;
out->del_recv_num = instance->statistic.del_recv_num;
out->del_succ_num = instance->statistic.del_succ_num;
out->del_error_num= instance->statistic.del_error_num;
out->session_num = instance->statistic.session_num; out->session_num = instance->statistic.session_num;
out->memory_used = instance->statistic.memory_used; out->memory_used = instance->statistic.memory_used;
} }
@@ -67,30 +70,42 @@ const char *tango_cache_get_object_key(struct tango_cache_ctx *ctx)
static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, enum CACHE_ERR_CODE error_code, struct cache_statistics *statistic) static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, enum CACHE_ERR_CODE error_code, struct cache_statistics *statistic)
{ {
if(method == CACHE_REQUEST_PUT) switch(method)
{ {
if(fail_state) case CACHE_REQUEST_PUT:
{ if(fail_state)
statistic->put_error_num += 1; {
} statistic->put_error_num += 1;
else }
{
statistic->put_succ_num += 1;
}
}
else
{
if(fail_state)
{
if(error_code == CACHE_ERR_CURL)
statistic->get_error_num += 1;
else else
statistic->get_miss_num += 1; {
} statistic->put_succ_num += 1;
else }
{ break;
statistic->get_succ_num += 1; case CACHE_REQUEST_GET:
} if(fail_state)
{
if(error_code == CACHE_ERR_CURL)
statistic->get_error_num += 1;
else
statistic->get_miss_num += 1;
}
else
{
statistic->get_succ_num += 1;
}
break;
case CACHE_REQUEST_DELETE:
if(fail_state)
{
statistic->del_error_num += 1;
}
else
{
statistic->del_succ_num += 1;
}
break;
default:break;
} }
} }
@@ -119,6 +134,8 @@ void easy_string_savedata(struct easy_string *estr, const char *data, size_t len
void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx) void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx)
{ {
struct multipart_etag_list *etag;
if(ctx->curl != NULL) if(ctx->curl != NULL)
{ {
curl_multi_remove_handle(ctx->instance->multi_hd, ctx->curl); curl_multi_remove_handle(ctx->instance->multi_hd, ctx->curl);
@@ -126,37 +143,37 @@ void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx)
} }
easy_string_destroy(&ctx->response); easy_string_destroy(&ctx->response);
if(ctx->method == CACHE_REQUEST_GET) switch(ctx->method)
{ {
easy_string_destroy(&ctx->response_tag); case CACHE_REQUEST_GET:
} easy_string_destroy(&ctx->get.response_tag);
else break;
{
struct multipart_etag_list *etag;
if(ctx->uploadID != NULL) free(ctx->uploadID); case CACHE_REQUEST_PUT:
if(ctx->combine_xml != NULL) free(ctx->combine_xml); if(ctx->put.uploadID != NULL) free(ctx->put.uploadID);
if(ctx->headers != NULL) curl_slist_free_all(ctx->headers); if(ctx->put.combine_xml != NULL) free(ctx->put.combine_xml);
if(ctx->evbuffer!=NULL) evbuffer_free(ctx->evbuffer); if(ctx->headers != NULL) curl_slist_free_all(ctx->headers);
if(ctx->put.evbuf!=NULL) evbuffer_free(ctx->put.evbuf);
TAILQ_FOREACH(etag, &ctx->cache_head, node) TAILQ_FOREACH(etag, &ctx->put.etag_head, node)
{
TAILQ_REMOVE(&ctx->cache_head, etag, node);
free(etag->etag);
free(etag);
}
if(ctx->future != NULL)
{
if(ctx->fail_state)
{ {
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, ctx->error); TAILQ_REMOVE(&ctx->put.etag_head, etag, node);
} free(etag->etag);
else free(etag);
}//no break here
case CACHE_REQUEST_DELETE:
if(ctx->future != NULL)
{ {
promise_success(future_to_promise(ctx->future), NULL); if(ctx->fail_state)
{
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, ctx->error);
}
else
{
promise_success(future_to_promise(ctx->future), NULL);
}
} }
} break;
default: break;
} }
update_statistics(ctx->method, ctx->fail_state, ctx->error_code, &ctx->instance->statistic); update_statistics(ctx->method, ctx->fail_state, ctx->error_code, &ctx->instance->statistic);
free(ctx); free(ctx);
@@ -173,12 +190,12 @@ int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data,
{ {
return -1; return -1;
} }
if(evbuffer_add(ctx->evbuffer, data, size)) if(evbuffer_add(ctx->put.evbuf, data, size))
{ {
return -1; return -1;
} }
ctx->instance->statistic.memory_used += size; ctx->instance->statistic.memory_used += size;
if(evbuffer_get_length(ctx->evbuffer) >= ctx->instance->upload_block_size) if(evbuffer_get_length(ctx->put.evbuf) >= ctx->instance->upload_block_size)
{ {
cache_kick_upload_minio_multipart(ctx, ctx->instance->upload_block_size); cache_kick_upload_minio_multipart(ctx, ctx->instance->upload_block_size);
} }
@@ -197,20 +214,20 @@ int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COP
size = evbuffer_get_length(evbuf); size = evbuffer_get_length(evbuf);
if(way == EVBUFFER_MOVE) if(way == EVBUFFER_MOVE)
{ {
if(evbuffer_add_buffer(ctx->evbuffer, evbuf)) if(evbuffer_add_buffer(ctx->put.evbuf, evbuf))
{ {
return -1; return -1;
} }
} }
else else
{ {
if(evbuffer_add_buffer_reference(ctx->evbuffer, evbuf)) if(evbuffer_add_buffer_reference(ctx->put.evbuf, evbuf))
{ {
return -1; return -1;
} }
} }
ctx->instance->statistic.memory_used += size; ctx->instance->statistic.memory_used += size;
if(evbuffer_get_length(ctx->evbuffer) >= ctx->instance->upload_block_size) if(evbuffer_get_length(ctx->put.evbuf) >= ctx->instance->upload_block_size)
{ {
cache_kick_upload_minio_multipart(ctx, ctx->instance->upload_block_size); cache_kick_upload_minio_multipart(ctx, ctx->instance->upload_block_size);
} }
@@ -296,8 +313,8 @@ struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *in
return NULL; return NULL;
} }
ctx->evbuffer = evbuffer_new(); ctx->put.evbuf = evbuffer_new();
TAILQ_INIT(&ctx->cache_head); TAILQ_INIT(&ctx->put.etag_head);
return ctx; return ctx;
} }
@@ -346,9 +363,9 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i
ctx->instance = instance; ctx->instance = instance;
ctx->future = future; ctx->future = future;
ctx->method = CACHE_REQUEST_GET; ctx->method = CACHE_REQUEST_GET;
ctx->get_state = GET_STATE_START; ctx->get.state = GET_STATE_START;
ctx->max_age = meta->get.max_age; ctx->get.max_age = meta->get.max_age;
ctx->min_fresh = meta->get.min_fresh; ctx->get.min_fresh = meta->get.min_fresh;
instance->statistic.get_recv_num += 1; instance->statistic.get_recv_num += 1;
if(instance->hash_object_key) if(instance->hash_object_key)
@@ -360,15 +377,42 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i
{ {
snprintf(ctx->object_key, 256, "%s", meta->url); snprintf(ctx->object_key, 256, "%s", meta->url);
} }
return ctx; return ctx;
} }
int tango_cache_fetch(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta) int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta)
{ {
return tango_cache_fetch_start(tango_cache_fetch_prepare(instance, future, meta)); return tango_cache_fetch_start(tango_cache_fetch_prepare(instance, future, meta));
} }
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* future, const char *objkey)
{
struct tango_cache_ctx *ctx;
char sha256[72]={0};
ctx = (struct tango_cache_ctx *)calloc(1, sizeof(struct tango_cache_ctx));
ctx->instance = instance;
ctx->future = future;
ctx->method = CACHE_REQUEST_DELETE;
instance->statistic.del_recv_num += 1;
if(instance->hash_object_key)
{
caculate_sha256(objkey, strlen(objkey), sha256, 72);
snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
}
else
{
snprintf(ctx->object_key, 256, "%s", objkey);
}
return ctx;
}
int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* future, const char *objkey)
{
return (cache_delete_minio_object(tango_cache_delete_prepare(instance, future, objkey))==1)?0:-1;
}
static void check_multi_info(CURLM *multi) static void check_multi_info(CURLM *multi)
{ {
CURLMsg *msg; CURLMsg *msg;
@@ -394,13 +438,18 @@ static void check_multi_info(CURLM *multi)
ctx->curl = NULL; ctx->curl = NULL;
ctx->res_code = 0; ctx->res_code = 0;
if(ctx->method == CACHE_REQUEST_GET) switch(ctx->method)
{ {
tango_cache_curl_get_done(easy, ctx, res, res_code); case CACHE_REQUEST_GET:
} tango_cache_curl_get_done(ctx, res, res_code);
else break;
{ case CACHE_REQUEST_PUT:
tango_cache_curl_put_done(easy, ctx, res, res_code); tango_cache_curl_put_done(ctx, res, res_code);
break;
case CACHE_REQUEST_DELETE:
tango_cache_curl_del_done(ctx, res, res_code);
break;
default: break;
} }
} }
} }
@@ -519,19 +568,13 @@ static int load_local_configure(struct tango_cache_instance *instance, const cha
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] MINIO_BROKERS_LIST not found.\n", profile_path, section); MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] MINIO_BROKERS_LIST not found.\n", profile_path, section);
return -1; return -1;
} }
MESA_load_profile_uint_def(profile_path, section, "CACHE_BLOCK_MAX_SIZE", &instance->block_len, 8192);
if(instance->block_len > 16777216)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_BLOCK_MAX_SIZE too large, must smaller than 16777216(16MB).\n", profile_path, section);
return -1;
}
MESA_load_profile_uint_def(profile_path, section, "CACHE_UPLOAD_BLOCK_SIZE", &instance->upload_block_size, 5242880); MESA_load_profile_uint_def(profile_path, section, "CACHE_UPLOAD_BLOCK_SIZE", &instance->upload_block_size, 5242880);
if(instance->upload_block_size < 5242880) if(instance->upload_block_size < 5242880)
{ {
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_UPLOAD_BLOCK_SIZE too small, must bigger than 5242880(5MB).\n", profile_path, section); MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_UPLOAD_BLOCK_SIZE too small, must bigger than 5242880(5MB).\n", profile_path, section);
return -1; return -1;
} }
MESA_load_profile_uint_def(profile_path, section, "CACHE_DEFAULT_TTL_SECOND", &intval, 3600); MESA_load_profile_uint_def(profile_path, section, "CACHE_DEFAULT_TTL_SECOND", &intval, 999999999);
if(intval < 60) if(intval < 60)
{ {
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_DEFAULT_TTL_SECOND too small, must bigger than 60s.\n", profile_path, section); MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_DEFAULT_TTL_SECOND too small, must bigger than 60s.\n", profile_path, section);

58
cache/tango_cache_client_in.h vendored
View File

@@ -17,6 +17,7 @@ enum CACHE_REQUEST_METHOD
{ {
CACHE_REQUEST_GET=0, CACHE_REQUEST_GET=0,
CACHE_REQUEST_PUT, CACHE_REQUEST_PUT,
CACHE_REQUEST_DELETE,
}; };
enum GET_OBJECT_STATE enum GET_OBJECT_STATE
@@ -54,7 +55,6 @@ struct tango_cache_instance
time_t relative_ttl; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ч<EFBFBD><D0A7> time_t relative_ttl; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ч<EFBFBD><D0A7>
u_int64_t cache_limit_size; u_int64_t cache_limit_size;
long max_cnn_host; long max_cnn_host;
u_int32_t block_len; //<2F><><EFBFBD><EFBFBD>buffercache<68>ڴ<EFBFBD><DAB4>Ļ<EFBFBD><C4BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1>ÿ<EFBFBD><C3BF>update<74><65><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD>ò<EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ӿ<EFBFBD><D3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
u_int32_t upload_block_size; //minio<69>ֶ<EFBFBD><D6B6>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD><EFBFBD> u_int32_t upload_block_size; //minio<69>ֶ<EFBFBD><D6B6>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD><EFBFBD>
enum CACHE_ERR_CODE error_code; enum CACHE_ERR_CODE error_code;
u_int32_t hash_object_key; u_int32_t hash_object_key;
@@ -67,6 +67,30 @@ struct multipart_etag_list
TAILQ_ENTRY(multipart_etag_list) node; TAILQ_ENTRY(multipart_etag_list) node;
}; };
struct cache_ctx_data_get
{
time_t max_age;//Get
time_t min_fresh;//Get
time_t expires;
time_t last_modify;
u_int32_t need_hdrs;
enum GET_OBJECT_STATE state;
struct easy_string response_tag;
};
struct cache_ctx_data_put
{
struct evbuffer *evbuf;
size_t upload_length;
size_t upload_offset;
char *uploadID;
char *combine_xml;
TAILQ_HEAD(__etag_list_head, multipart_etag_list) etag_head;
enum PUT_OBJECT_STATE state;
u_int32_t part_index; //<2F><>RESPONSE_HDR_
bool close_state; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ùر<C3B9>
};
struct tango_cache_ctx struct tango_cache_ctx
{ {
CURL *curl; CURL *curl;
@@ -74,34 +98,18 @@ struct tango_cache_ctx
struct future* future; struct future* future;
char error[CURL_ERROR_SIZE]; char error[CURL_ERROR_SIZE];
char object_key[256]; char object_key[256];
char hostport[24]; //<2F><>ͬctxʹ<78><CAB9><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC>IP<49><50><EFBFBD><EFBFBD>֤pipeline˳<65><CBB3><EFBFBD><EFBFBD>
enum CACHE_REQUEST_METHOD method; enum CACHE_REQUEST_METHOD method;
enum CACHE_ERR_CODE error_code; enum CACHE_ERR_CODE error_code;
struct evbuffer *evbuffer;
union{
enum PUT_OBJECT_STATE put_state;
enum GET_OBJECT_STATE get_state;
};
u_int32_t part_index;
u_int32_t need_hdrs; //<2F><>RESPONSE_HDR_
bool fail_state;
bool close_state; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ùر<C3B9>
long res_code;
time_t max_age;//Get
time_t min_fresh;//Get
time_t expires;
time_t last_modify;
size_t upload_length;
size_t upload_offset;
char *uploadID;
char *combine_xml;
struct easy_string response; struct easy_string response;
struct easy_string response_tag;
TAILQ_HEAD(__etag_list_head, multipart_etag_list) cache_head;
bool fail_state;
long res_code;
union{
struct cache_ctx_data_put put;
struct cache_ctx_data_get get;
};
struct tango_cache_instance *instance; struct tango_cache_instance *instance;
}; };
@@ -111,13 +119,13 @@ struct curl_socket_data
}; };
void easy_string_savedata(struct easy_string *estr, const char *data, size_t len); void easy_string_savedata(struct easy_string *estr, const char *data, size_t len);
void easy_string_expand(struct easy_string *estr, size_t to_size);
void easy_string_destroy(struct easy_string *estr); void easy_string_destroy(struct easy_string *estr);
void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx); void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx);
struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta); struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta); struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* future, const char *objkey);
#endif #endif

208
cache/tango_cache_transfer.cpp vendored
View File

@@ -36,10 +36,10 @@ static size_t curl_put_multipart_header_cb(void *ptr, size_t size, size_t count,
etag = (struct multipart_etag_list *)malloc(sizeof(struct multipart_etag_list)); etag = (struct multipart_etag_list *)malloc(sizeof(struct multipart_etag_list));
totallen = end - start + 1; totallen = end - start + 1;
etag->etag = (char *)malloc(totallen + 1); etag->etag = (char *)malloc(totallen + 1);
etag->part_number = ctx->part_index; etag->part_number = ctx->put.part_index;
memcpy(etag->etag, start, totallen); memcpy(etag->etag, start, totallen);
*(etag->etag + totallen) = '\0'; *(etag->etag + totallen) = '\0';
TAILQ_INSERT_TAIL(&ctx->cache_head, etag, node); TAILQ_INSERT_TAIL(&ctx->put.etag_head, etag, node);
} }
} }
@@ -78,19 +78,19 @@ static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, v
size_t len, space=size*count, send_len; size_t len, space=size*count, send_len;
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
if(size==0 || count==0 || ctx->upload_offset>=ctx->upload_length) if(size==0 || count==0 || ctx->put.upload_offset>=ctx->put.upload_length)
{ {
return 0; return 0;
} }
len = ctx->upload_length - ctx->upload_offset; len = ctx->put.upload_length - ctx->put.upload_offset;
if(len > space) if(len > space)
{ {
len = space; len = space;
} }
send_len = evbuffer_remove(ctx->evbuffer, ptr, len); send_len = evbuffer_remove(ctx->put.evbuf, ptr, len);
assert(send_len>0); assert(send_len>0);
ctx->upload_offset += send_len; ctx->put.upload_offset += send_len;
ctx->instance->statistic.memory_used -= send_len; ctx->instance->statistic.memory_used -= send_len;
return send_len; return send_len;
@@ -107,14 +107,14 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
return -1; return -1;
} }
ctx->upload_offset = 0; ctx->put.upload_offset = 0;
if(full) if(full)
{ {
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key); snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
} }
else else
{ {
snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ++ctx->part_index, ctx->uploadID); snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ++ctx->put.part_index, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb); curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
} }
@@ -132,7 +132,7 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L); curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L); curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->upload_length); curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put.upload_length);
curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb); curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb);
curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx); curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);
@@ -199,14 +199,14 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
return 1; return 1;
} }
bool cache_delete_minio_object(struct tango_cache_ctx *ctx) int cache_delete_minio_object(struct tango_cache_ctx *ctx)
{ {
CURLMcode rc; CURLMcode rc;
char minio_url[256]; char minio_url[256];
if(NULL == (ctx->curl=curl_easy_init())) if(NULL == (ctx->curl=curl_easy_init()))
{ {
return false; return 0;
} }
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key); snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
@@ -223,7 +223,7 @@ bool cache_delete_minio_object(struct tango_cache_ctx *ctx)
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK); assert(rc==CURLM_OK);
return true; return 1;
} }
//return value: true-<2D>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD><C2BC><EFBFBD>false-δ<><CEB4><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD> //return value: true-<2D>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD><C2BC><EFBFBD>false-δ<><CEB4><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD>
@@ -237,7 +237,7 @@ bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx)
return false; return false;
} }
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID); snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -265,9 +265,9 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
{ {
return false; return false;
} }
construct_complete_xml(ctx, &ctx->combine_xml, &len); construct_complete_xml(ctx, &ctx->put.combine_xml, &len);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID); snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -279,7 +279,7 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(ctx->curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->combine_xml); curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->put.combine_xml);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, len); //<2F><><EFBFBD><EFBFBD>Content-Length curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, len); //<2F><><EFBFBD><EFBFBD>Content-Length
if(ctx->headers != NULL) if(ctx->headers != NULL)
@@ -300,17 +300,17 @@ bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block
{ {
int ret = 1; int ret = 1;
switch(ctx->put_state) switch(ctx->put.state)
{ {
case PUT_STATE_START: case PUT_STATE_START:
ctx->put_state = PUT_STATE_WAIT_START; ctx->put.state = PUT_STATE_WAIT_START;
ret = curl_get_minio_uploadID(ctx); ret = curl_get_minio_uploadID(ctx);
break; break;
case PUT_STATE_PART: case PUT_STATE_PART:
if(ctx->curl == NULL) if(ctx->curl == NULL)
{ {
ctx->upload_length = block_len; ctx->put.upload_length = block_len;
ret = http_put_bodypart_request_evbuf(ctx, false); ret = http_put_bodypart_request_evbuf(ctx, false);
} }
break; break;
@@ -331,9 +331,9 @@ int http_put_complete_part_evbuf(struct tango_cache_ctx *ctx)
{ {
int ret=0; int ret=0;
ctx->put_state = PUT_STATE_END; ctx->put.state = PUT_STATE_END;
ctx->upload_length = evbuffer_get_length(ctx->evbuffer); ctx->put.upload_length = evbuffer_get_length(ctx->put.evbuf);
if(ctx->upload_length > 0) if(ctx->put.upload_length > 0)
{ {
ret = http_put_bodypart_request_evbuf(ctx, true); ret = http_put_bodypart_request_evbuf(ctx, true);
if(ret <= 0) if(ret <= 0)
@@ -355,14 +355,14 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
{ {
int ret = 0; int ret = 0;
ctx->close_state = true;//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>رգ<D8B1><D5A3>ڲ<EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD>ת<EFBFBD><D7AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ٹر<D9B9> ctx->put.close_state = true;//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>رգ<D8B1><D5A3>ڲ<EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD>ת<EFBFBD><D7AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ٹر<D9B9>
if(ctx->fail_state) if(ctx->fail_state)
{ {
tango_cache_ctx_destroy(ctx); tango_cache_ctx_destroy(ctx);
return 0; return 0;
} }
switch(ctx->put_state) switch(ctx->put.state)
{ {
case PUT_STATE_START: case PUT_STATE_START:
http_put_complete_part_evbuf(ctx); http_put_complete_part_evbuf(ctx);
@@ -371,12 +371,12 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
case PUT_STATE_PART: case PUT_STATE_PART:
if(ctx->curl == NULL) if(ctx->curl == NULL)
{ {
ctx->upload_length = evbuffer_get_length(ctx->evbuffer); ctx->put.upload_length = evbuffer_get_length(ctx->put.evbuf);
if(ctx->upload_length == 0) if(ctx->put.upload_length == 0)
{ {
if(cache_kick_combine_minio(ctx)) if(cache_kick_combine_minio(ctx))
{ {
ctx->put_state = PUT_STATE_END; ctx->put.state = PUT_STATE_END;
} }
else else
{ {
@@ -394,7 +394,7 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
ctx->error_code = CACHE_ERR_CURL; ctx->error_code = CACHE_ERR_CURL;
if(cache_cancel_upload_minio(ctx)) if(cache_cancel_upload_minio(ctx))
{ {
ctx->put_state = PUT_STATE_CANCEL; ctx->put.state = PUT_STATE_CANCEL;
} }
else else
{ {
@@ -413,18 +413,18 @@ int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx)
return ret; return ret;
} }
void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code) void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{ {
switch(ctx->put_state) switch(ctx->put.state)
{ {
case PUT_STATE_WAIT_START: case PUT_STATE_WAIT_START:
if(res!=CURLE_OK||res_code!=200L|| ctx->fail_state || !parse_uploadID_xml(ctx->response.buff, ctx->response.len, &ctx->uploadID)) if(res!=CURLE_OK||res_code!=200L|| ctx->fail_state || !parse_uploadID_xml(ctx->response.buff, ctx->response.len, &ctx->put.uploadID))
{ {
easy_string_destroy(&ctx->response); easy_string_destroy(&ctx->response);
ctx->error_code = CACHE_ERR_CURL; ctx->error_code = CACHE_ERR_CURL;
ctx->fail_state = true; ctx->fail_state = true;
if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error); if(res != CURLE_OK) MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "%s", ctx->error);
if(ctx->close_state) if(ctx->put.close_state)
{ {
tango_cache_ctx_destroy(ctx); tango_cache_ctx_destroy(ctx);
} }
@@ -432,14 +432,14 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
else else
{ {
easy_string_destroy(&ctx->response); easy_string_destroy(&ctx->response);
ctx->put_state = PUT_STATE_PART; ctx->put.state = PUT_STATE_PART;
if(ctx->close_state) if(ctx->put.close_state)
{ {
cache_kick_upload_minio_end(ctx); cache_kick_upload_minio_end(ctx);
} }
else else
{ {
size_t upload_length = evbuffer_get_length(ctx->evbuffer); size_t upload_length = evbuffer_get_length(ctx->put.evbuf);
if(upload_length >= ctx->instance->upload_block_size) if(upload_length >= ctx->instance->upload_block_size)
{ {
cache_kick_upload_minio_multipart(ctx, upload_length); cache_kick_upload_minio_multipart(ctx, upload_length);
@@ -459,20 +459,20 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
{ {
if(cache_cancel_upload_minio(ctx)) if(cache_cancel_upload_minio(ctx))
{ {
ctx->put_state = PUT_STATE_CANCEL; ctx->put.state = PUT_STATE_CANCEL;
} }
else if(ctx->close_state) else if(ctx->put.close_state)
{ {
tango_cache_ctx_destroy(ctx); tango_cache_ctx_destroy(ctx);
} }
} }
else if(ctx->close_state) else if(ctx->put.close_state)
{ {
cache_kick_upload_minio_end(ctx); cache_kick_upload_minio_end(ctx);
} }
else else
{ {
size_t upload_length = evbuffer_get_length(ctx->evbuffer); size_t upload_length = evbuffer_get_length(ctx->put.evbuf);
if(upload_length >= ctx->instance->upload_block_size) if(upload_length >= ctx->instance->upload_block_size)
{ {
cache_kick_upload_minio_multipart(ctx, upload_length); cache_kick_upload_minio_multipart(ctx, upload_length);
@@ -481,7 +481,7 @@ void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
break; break;
case PUT_STATE_CANCEL: //<2F>ȴ<EFBFBD><C8B4>ر<EFBFBD> case PUT_STATE_CANCEL: //<2F>ȴ<EFBFBD><C8B4>ر<EFBFBD>
if(ctx->close_state) if(ctx->put.close_state)
{ {
tango_cache_ctx_destroy(ctx); tango_cache_ctx_destroy(ctx);
} }
@@ -514,7 +514,7 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM
} }
return -1; return -1;
} }
ctx->put_state = PUT_STATE_END; ctx->put.state = PUT_STATE_END;
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key); snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
@@ -556,18 +556,18 @@ int tango_cache_upload_once_start_evbuf(struct tango_cache_ctx *ctx, enum EVBUFF
{ {
size_t size; size_t size;
ctx->evbuffer = evbuffer_new(); ctx->put.evbuf = evbuffer_new();
size = evbuffer_get_length(evbuf); size = evbuffer_get_length(evbuf);
if(way == EVBUFFER_MOVE) if(way == EVBUFFER_MOVE)
{ {
if(evbuffer_add_buffer(ctx->evbuffer, evbuf)) if(evbuffer_add_buffer(ctx->put.evbuf, evbuf))
{ {
return -1; return -1;
} }
} }
else else
{ {
if(evbuffer_add_buffer_reference(ctx->evbuffer, evbuf)) if(evbuffer_add_buffer_reference(ctx->put.evbuf, evbuf))
{ {
return -1; return -1;
} }
@@ -577,9 +577,18 @@ int tango_cache_upload_once_start_evbuf(struct tango_cache_ctx *ctx, enum EVBUFF
return http_put_complete_part_evbuf(ctx); return http_put_complete_part_evbuf(ctx);
} }
void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code) void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{ {
switch(ctx->get_state) if(res!=CURLE_OK || (res_code!=204L && res_code!=200L ))
{
ctx->fail_state = true;
}
tango_cache_ctx_destroy(ctx);
}
void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
switch(ctx->get.state)
{ {
case GET_STATE_START: case GET_STATE_START:
if(!ctx->fail_state) if(!ctx->fail_state)
@@ -600,7 +609,7 @@ void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode
case GET_STATE_DELETE: case GET_STATE_DELETE:
if(cache_delete_minio_object(ctx)) if(cache_delete_minio_object(ctx))
{ {
ctx->get_state = GET_STATE_END; ctx->get.state = GET_STATE_END;
} }
else else
{ {
@@ -620,27 +629,28 @@ static size_t curl_get_response_body_cb(void *ptr, size_t size, size_t count, vo
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
struct tango_cache_result result; struct tango_cache_result result;
if(ctx->fail_state || ctx->get_state==GET_STATE_DELETE) if(ctx->fail_state || ctx->get.state==GET_STATE_DELETE)
{ {
return size*count; return size*count;
} }
if(ctx->need_hdrs!=RESPONSE_HDR_ALL) //<2F><>Expiresʱ if(ctx->get.need_hdrs!=RESPONSE_HDR_ALL) //<2F><>Expiresʱ
{ {
ctx->fail_state = true; ctx->fail_state = true;
ctx->error_code = CACHE_ERR_CURL; ctx->error_code = CACHE_ERR_CURL;
ctx->get_state = GET_STATE_DELETE; ctx->get.state = GET_STATE_DELETE;
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache Expires or last-modify not found"); ctx->instance->statistic.del_recv_num += 1;
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache Expires or x-amz-meta-lm not found");
return size*count; return size*count;
} }
if(ctx->response_tag.len > 0) if(ctx->get.response_tag.len > 0)
{ {
result.data_frag = ctx->response_tag.buff; result.data_frag = ctx->get.response_tag.buff;
result.size = ctx->response_tag.len; result.size = ctx->get.response_tag.len;
result.type = RESULT_TYPE_USERTAG; result.type = RESULT_TYPE_USERTAG;
promise_success(future_to_promise(ctx->future), &result); promise_success(future_to_promise(ctx->future), &result);
easy_string_destroy(&ctx->response_tag); easy_string_destroy(&ctx->get.response_tag);
} }
if(ctx->response.len > 0) if(ctx->response.len > 0)
{ {
@@ -661,14 +671,15 @@ static bool check_expires_header(struct tango_cache_ctx *ctx, const char *expire
{ {
time_t time_gmt; time_t time_gmt;
ctx->expires = expires_hdr2timestamp(expires_val, len); ctx->get.expires = expires_hdr2timestamp(expires_val, len);
time_gmt = get_gmtime_timestamp(time(NULL)); time_gmt = get_gmtime_timestamp(time(NULL));
if(time_gmt > ctx->expires) //<2F><><EFBFBD><EFBFBD>ʧЧ<CAA7><D0A7>TODO relative_age<67>ĺ<EFBFBD><C4BA><EFBFBD><EFBFBD><EFBFBD>ɶ if(time_gmt > ctx->get.expires) //<2F><><EFBFBD><EFBFBD>ʧЧ<CAA7><D0A7>TODO relative_age<67>ĺ<EFBFBD><C4BA><EFBFBD><EFBFBD><EFBFBD>ɶ
{ {
ctx->fail_state = true; ctx->fail_state = true;
ctx->error_code = CACHE_TIMEOUT; ctx->error_code = CACHE_TIMEOUT;
ctx->get_state = GET_STATE_DELETE; //<2F><><EFBFBD><EFBFBD>ʧЧʱ<D0A7><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD> ctx->get.state = GET_STATE_DELETE; //<2F><><EFBFBD><EFBFBD>ʧЧʱ<D0A7><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
ctx->instance->statistic.del_recv_num += 1;
easy_string_destroy(&ctx->response); easy_string_destroy(&ctx->response);
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not fresh"); promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, "cache not fresh");
return false; return false;
@@ -680,11 +691,11 @@ static bool check_fresh_header(struct tango_cache_ctx *ctx)
{ {
time_t now_gmt; time_t now_gmt;
if(ctx->need_hdrs != RESPONSE_HDR_ALL) if(ctx->get.need_hdrs != RESPONSE_HDR_ALL)
return true; return true;
now_gmt = get_gmtime_timestamp(time(NULL)); now_gmt = get_gmtime_timestamp(time(NULL));
if(ctx->last_modify+ctx->max_age > now_gmt || now_gmt+ctx->min_fresh>ctx->expires) if(ctx->get.last_modify+ctx->get.max_age > now_gmt || now_gmt+ctx->get.min_fresh>ctx->get.expires)
{ {
ctx->fail_state = true; ctx->fail_state = true;
ctx->error_code = CACHE_TIMEOUT; ctx->error_code = CACHE_TIMEOUT;
@@ -716,8 +727,9 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
char *start=(char *)ptr, *pos_colon; char *start=(char *)ptr, *pos_colon;
size_t raw_len = size*count, hdrlen=size*count; size_t raw_len = size*count, hdrlen=size*count;
char usertag[2048]; char usertag[2048];
size_t datalen;
if(ctx->fail_state || ctx->get_state==GET_STATE_DELETE) if(ctx->fail_state || ctx->get.state==GET_STATE_DELETE)
{ {
return raw_len; return raw_len;
} }
@@ -725,49 +737,51 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
{ {
return raw_len; return raw_len;
} }
pos_colon = (char*)memchr(start, ':', raw_len);
if((pos_colon=(char*)memchr(start, ':', raw_len))!=NULL) if(pos_colon == NULL)
{ {
size_t datalen = pos_colon - start; return raw_len;
switch(datalen) }
{
case 7: datalen = pos_colon - start;
if(strcmp_one_word_mesa_equal_len("expires", "EXPIRES", start, 7)) switch(datalen)
{
case 7:
if(strcmp_one_word_mesa_equal_len("expires", "EXPIRES", start, 7))
{
ctx->get.need_hdrs |= RESPONSE_HDR_EXPIRES;
if(!check_expires_header(ctx, pos_colon + 1, raw_len - datalen - 1) || !check_fresh_header(ctx))
{ {
ctx->need_hdrs |= RESPONSE_HDR_EXPIRES; return raw_len;
if(!check_expires_header(ctx, pos_colon + 1, raw_len - datalen - 1) || !check_fresh_header(ctx))
{
return raw_len;
}
} }
break; }
case 13: break;
if(strcmp_one_word_mesa_equal_len("x-amz-meta-lm", "X-AMZ-META-LM", start, 13)) case 13:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-lm", "X-AMZ-META-LM", start, 13))
{
ctx->get.need_hdrs |= RESPONSE_HDR_LAST_MOD;
sscanf(pos_colon+1, "%lu", &ctx->get.last_modify);
if(!check_fresh_header(ctx))
{ {
ctx->need_hdrs |= RESPONSE_HDR_LAST_MOD; return raw_len;
sscanf(pos_colon+1, "%lu", &ctx->last_modify);
if(!check_fresh_header(ctx))
{
return raw_len;
}
} }
break; }
case 15: break;
if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15)) case 15:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15))
{
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0)
{ {
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0) easy_string_savedata(&ctx->get.response_tag, usertag, hdrlen);
{
easy_string_savedata(&ctx->response_tag, usertag, hdrlen);
}
} }
break; }
case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; break;
case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 14: if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; case 14: if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
default: break; case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
} default: break;
} }
return raw_len; return raw_len;
} }

7
cache/tango_cache_transfer.h vendored
View File

@@ -6,10 +6,11 @@
#include "tango_cache_client_in.h" #include "tango_cache_client_in.h"
void tango_cache_curl_put_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code); void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_get_done(CURL *easy, struct tango_cache_ctx *ctx, CURLcode res, long res_code); void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
bool cache_delete_minio_object(struct tango_cache_ctx *ctx); int cache_delete_minio_object(struct tango_cache_ctx *ctx);
int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx); int cache_kick_upload_minio_end(struct tango_cache_ctx *ctx);
bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block_len); bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block_len);

2
cache/tango_cache_xml.cpp vendored
View File

@@ -58,7 +58,7 @@ int construct_complete_xml(struct tango_cache_ctx *ctx, char **xml, int *len)
xmlNewProp(root, (const xmlChar *)"xmlns",(const xmlChar *)"http://s3.amazonaws.com/doc/2006-03-01/"); xmlNewProp(root, (const xmlChar *)"xmlns",(const xmlChar *)"http://s3.amazonaws.com/doc/2006-03-01/");
xmlDocSetRootElement(pdoc, root); xmlDocSetRootElement(pdoc, root);
TAILQ_FOREACH(etag, &ctx->cache_head, node) TAILQ_FOREACH(etag, &ctx->put.etag_head, node)
{ {
sprintf(number, "%u", etag->part_number); sprintf(number, "%u", etag->part_number);
child = xmlNewChild(root, NULL, (const xmlChar*)"Part", NULL); child = xmlNewChild(root, NULL, (const xmlChar*)"Part", NULL);

51
cache/test_demo/cache_evbase_test.cpp vendored
View File

@@ -22,7 +22,7 @@
#include "cache_evbase_client.h" #include "cache_evbase_client.h"
struct cache_evbase_instance *instance_asyn; struct cache_evbase_instance *instance_asyn;
int still_runing=1; int runing_over=0;
struct future_pdata struct future_pdata
{ {
@@ -59,15 +59,14 @@ void get_future_success(future_result_t* result, void * user)
future_destroy(pdata->future); future_destroy(pdata->future);
fclose(pdata->fp); fclose(pdata->fp);
free(pdata); free(pdata);
still_runing = 0; runing_over = 1;
} }
} }
void get_future_failed(enum e_future_error err, const char * what, void * user) void get_future_failed(enum e_future_error err, const char * what, void * user)
{ {
printf("GET fail: %s\n", what); printf("GET fail: %s\n", what);
sleep(5); runing_over = 2;
still_runing = 0;
} }
void put_future_success(future_result_t* result, void * user) void put_future_success(future_result_t* result, void * user)
@@ -77,7 +76,7 @@ void put_future_success(future_result_t* result, void * user)
printf("PUT %s succ\n", pdata->filename); printf("PUT %s succ\n", pdata->filename);
future_destroy(pdata->future); future_destroy(pdata->future);
free(pdata); free(pdata);
still_runing = 0; runing_over = 1;
} }
void put_future_failed(enum e_future_error err, const char * what, void * user) void put_future_failed(enum e_future_error err, const char * what, void * user)
{ {
@@ -86,7 +85,26 @@ void put_future_failed(enum e_future_error err, const char * what, void * user)
printf("PUT %s fail: %s\n", what, pdata->filename); printf("PUT %s fail: %s\n", what, pdata->filename);
future_destroy(pdata->future); future_destroy(pdata->future);
free(pdata); free(pdata);
still_runing = 0; runing_over = 1;
}
void del_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
void del_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s fail: %s\n", pdata->filename, what);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
} }
char * get_file_content(const char *filename, size_t *filelen_out) char * get_file_content(const char *filename, size_t *filelen_out)
@@ -139,7 +157,7 @@ int main(int argc, char **argv)
if(argc != 2 && argc!=3) if(argc != 2 && argc!=3)
{ {
printf("USGAE: %s <PUT/PUTONCE/PUTONCEEV/GET:filename> [get_out_file_index]\n", argv[0]); printf("USGAE: %s <PUT/PUTONCE/PUTONCEEV/GET/DEL:filename> [get_out_file_index]\n", argv[0]);
return -1; return -1;
} }
if(argc==3) if(argc==3)
@@ -182,7 +200,14 @@ int main(int argc, char **argv)
promise_set_ctx(future_to_promise(pdata->future), NULL, NULL); promise_set_ctx(future_to_promise(pdata->future), NULL, NULL);
pdata->fp = fopen(filename_out, "w"); pdata->fp = fopen(filename_out, "w");
cache_evbase_fetch(instance_asyn, pdata->future, &meta); cache_evbase_fetch_object(instance_asyn, pdata->future, &meta);
}
else if(!strcasecmp(p, "DEL"))
{
pdata->future = future_create(del_future_success, del_future_failed, pdata);
promise_set_ctx(future_to_promise(pdata->future), NULL, NULL);
sprintf(pdata->filename, "%s", filename_in);
cache_evbase_delete_object(instance_asyn, pdata->future, filename_in);
} }
else if(!strcasecmp(p, "PUTONCE")) else if(!strcasecmp(p, "PUTONCE"))
{ {
@@ -237,7 +262,7 @@ int main(int argc, char **argv)
printf("Waiting to finish.......\n"); printf("Waiting to finish.......\n");
static int num=0; static int num=0;
while(still_runing) while(!runing_over)
{ {
/*if(++num==10) /*if(++num==10)
{ {
@@ -245,11 +270,15 @@ int main(int argc, char **argv)
}*/ }*/
sleep(1); sleep(1);
} }
if(runing_over==2) //GETʱ<54><CAB1>ʱɾ<CAB1><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
{
sleep(5);
}
struct cache_statistics out; struct cache_statistics out;
cache_evbase_get_statistics(instance_asyn, &out); cache_evbase_get_statistics(instance_asyn, &out);
printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, session: %llu, memory: %llu\n", printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, del_recv: %llu, del_succ: %llu, del_fail: %llu, session: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.session_num, out.memory_used); out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.del_recv_num, out.del_succ_num, out.del_error_num, out.session_num, out.memory_used);
return 0; return 0;
} }

3
cache/test_demo/pangu_tg_cahce.conf vendored
View File

@@ -8,9 +8,6 @@ MAX_CONNECTION_PER_HOST=10
#bucket的名称 #bucket的名称
CACHE_BUCKET_NAME=openbucket CACHE_BUCKET_NAME=openbucket
#内部分配缓存块的大小，根据上传的文件大小分布而定，最大16777216（16MB）
CACHE_BLOCK_MAX_SIZE=8192
#缓存最大占用的内存空间大小，超出空间时上传失败 #缓存最大占用的内存空间大小，超出空间时上传失败
MAX_USED_MEMORY_SIZE_MB=5120 MAX_USED_MEMORY_SIZE_MB=5120

2
cache/test_demo/tango_cache_test.c vendored
View File

@@ -175,7 +175,7 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
pdata->future = future_create(get_future_success, get_future_failed, pdata); pdata->future = future_create(get_future_success, get_future_failed, pdata);
promise_set_ctx(future_to_promise(pdata->future), NULL, NULL); promise_set_ctx(future_to_promise(pdata->future), NULL, NULL);
tango_cache_fetch(tango_instance, pdata->future, &meta); tango_cache_fetch_object(tango_instance, pdata->future, &meta);
} }
else if(!strcasecmp(p, "PUTONCE")) else if(!strcasecmp(p, "PUTONCE"))
{ {