gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

对外API增加USERTAG

Browse Source
This commit is contained in:
zhangchengwei
2018-09-23 15:35:13 +08:00
committed by zhengchao
parent c974c534d0
commit 282eca63bd
10 changed files with 144 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cache/Makefile vendored
View File

@@ -6,7 +6,7 @@ LDFLAGS =
LIBS = -lMESA_handle_logger -lMESA_htable -lMESA_prof_load -lwiredcfg
LIBS += -lssl -lcrypto -lcurl -levent -lxml2
OBJS = tango_cache_tools.o tango_cache_client.o tango_cache_transfer.o tango_cache_xml.o cache_evbase_client.o
OBJS = tfe_future.o tango_cache_tools.o tango_cache_client.o tango_cache_transfer.o tango_cache_xml.o cache_evbase_client.o
TARGET_A=pangu_tango_cache.a

11
cache/cache_evbase_client.cpp vendored
View File

@@ -39,7 +39,7 @@ struct databuffer
enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn)
{
return ctx_asyn->ctx->error_code;
return tango_cache_get_last_error(ctx_asyn->ctx);
}
enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance)
{
@@ -51,6 +51,11 @@ void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, s
tango_cache_get_statistics(instance->instance, out);
}
struct tango_cache_result *cache_evbase_read_result(void *promise_result)
{
return tango_cache_read_result(promise_result);
}
const char *cache_evbase_get_object_key(struct cache_evbase_ctx *ctx_asyn)
{
return tango_cache_get_object_key(ctx_asyn->ctx);
@@ -351,7 +356,7 @@ int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct
}
if(objectkey != NULL)
{
snprintf(objectkey, keysize, "%s", ctx->file_key);
snprintf(objectkey, keysize, "%s", ctx->object_key);
}
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
@@ -402,7 +407,7 @@ int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struc
}
if(objectkey != NULL)
{
snprintf(objectkey, keysize, "%s", ctx->file_key);
snprintf(objectkey, keysize, "%s", ctx->object_key);
}
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));

21
cache/include/cache_evbase_client.h vendored
View File

@@ -20,18 +20,21 @@ struct cache_evbase_ctx
struct cache_evbase_instance *instance_asyn;
};
/*API<50><49>ʹ<EFBFBD><CAB9>˵<EFBFBD><CBB5><EFBFBD>ο<EFBFBD>tango_cache_client.h*/
enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn);
enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance);
void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out);
struct cache_evbase_instance *cache_evbase_instance_new(const char* profile_path, const char* section, void *runtimelog);
struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta);
int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size);
int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf);
void cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn);
const char *cache_evbase_get_object_key(struct cache_evbase_ctx *ctx_asyn);
//GET<45>ӿ<EFBFBD>
int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta);
struct tango_cache_result *cache_evbase_read_result(void *promise_result);
//һ<><D2BB><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD>
int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* future,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size,
struct tango_cache_meta *meta,
@@ -41,7 +44,13 @@ int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struc
struct tango_cache_meta *meta,
char *objectkey, size_t keysize);
int cache_evbase_fetch(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta);
//<2F><>ʽ<EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD>
struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta);
int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size);
int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf);
void cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn);
const char *cache_evbase_get_object_key(struct cache_evbase_ctx *ctx_asyn);
#endif

35
cache/include/tango_cache_client.h vendored
View File

@@ -7,6 +7,8 @@
#include "tfe_future.h"
#include "proxy_cache.h"
#define USER_TAG_MAX_LEN 1518
enum CACHE_ERR_CODE
{
CACHE_OK=0,
@@ -43,13 +45,14 @@ struct cache_statistics
enum CACHE_RESULT_TYPE
{
RESULT_TYPE_HEADER=0,
RESULT_TYPE_USERTAG,
RESULT_TYPE_BODY,
};
//promise_success<73>Ľ<EFBFBD><C4BD><EFBFBD>result
struct tango_cache_result
{
void *data_frag; //<2F><><EFBFBD><EFBFBD>typeΪRESULT_TYPE_HEADER<45><52><EFBFBD><EFBFBD><EFBFBD>ݻ<EFBFBD><DDBB><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
const void *data_frag; //<2F><><EFBFBD><EFBFBD>typeΪRESULT_TYPE_HEADER<45><52><EFBFBD><EFBFBD><EFBFBD>ݻ<EFBFBD><DDBB><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
size_t size;
enum CACHE_RESULT_TYPE type;
};
@@ -67,8 +70,9 @@ enum CACHE_HTTP_HDR_TYPE
struct tango_cache_meta
{
const char* url; //<2F><><EFBFBD><EFBFBD>:URL<52><4C><EFBFBD>ǽṹ<C7BD><E1B9B9><EFBFBD><EFBFBD>־:<3A>ļ<EFBFBD><C4BC><EFBFBD>
const char* std_hdr[HDR_CONTENT_NUM]; //<2F><><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"Content-Type:"<22><><EFBFBD><EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
const char* other_hdr; //<2F><><EFBFBD>󳤶Ȳ<EFBFBD><EFBFBD>ܳ<EFBFBD><EFBFBD><EFBFBD>1535<EFBFBD>ֽڣ<EFBFBD>GETʱ<EFBFBD><EFBFBD>ԭ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
const char* std_hdr[HDR_CONTENT_NUM]; //<2F><><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"Content-Type: text/html"<22><><EFBFBD><EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>У<EFBFBD>NULL<EFBFBD><EFBFBD>ʾû<EFBFBD>и<EFBFBD>ͷ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>
const char* usertag; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD>GETʱ<EFBFBD><EFBFBD>ԭ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
size_t usertag_len; //<2F><><EFBFBD>󳤶<EFBFBD>USER_TAG_MAX_LEN<45><4E>0<EFBFBD><30>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7>
union{
struct response_freshness put;//TODO
struct request_freshness get;
@@ -86,14 +90,25 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str
//ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>̴߳<DFB3><CCB4><EFBFBD>һ<EFBFBD><D2BB>instance
struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,const char* profile_path, const char* section, void *runtimelog);
//<2F>ɹ<EFBFBD>ʱ<EFBFBD>ص<EFBFBD>promise_success, resultΪNULLʱ<4C><CAB1>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
//ʧ<><CAA7>ʱ<EFBFBD>ص<EFBFBD>promise_failed(<28><>һ<EFBFBD><D2BB>)<29><>ʹ<EFBFBD><CAB9>get_last_error<6F><72>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD>룻
//future<72><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ΪNULL
int tango_cache_fetch(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
/*UPLOAD<41>ӿڵ<D3BF>API*/
//<2F><><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD>objectkey<65><79>Ϊ<EFBFBD>գ<EFBFBD><D5A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>KEY<45><59><EFBFBD><EFBFBD>CACHE_OBJECT_KEY_HASH_SWITCH=1<><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>ϣʱ<CFA3><CAB1><EFBFBD><EFBFBD>
//<2F><><EFBFBD><EFBFBD>0<EFBFBD><30>ʾ<EFBFBD>ɹ<EFBFBD><C9B9><EFBFBD><0<><30>ʾʧ<CABE>ܣ<EFBFBD><DCA3><EFBFBD>ͬ
//<2F><>promise_success<73><73>result<6C><74><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ҪCaller<65><72><EFBFBD><EFBFBD>tango_cache_result_release<73>ͷţ<CDB7>
struct tango_cache_result *tango_cache_read_result(void *promise_result);
/* UPLOAD<41>ӿڵ<D3BF>API
* ע<><D7A2>: UPLOAD<41>ӿڵ<D3BF>API<50><49><EFBFBD><EFBFBD>future<72><65>ΪNULL<4C><4C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>֪ͨ<CDA8>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>򲻵<EFBFBD><F2B2BBB5>ã<EFBFBD>
*/
/*<2A><><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>ϴ<EFBFBD>API*/
//<2F><>objectkey<65><79>Ϊ<EFBFBD>գ<EFBFBD><D5A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>KEY<45><59><EFBFBD><EFBFBD>CACHE_OBJECT_KEY_HASH_SWITCH=1<><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>ϣʱ<CFA3><CAB1><EFBFBD>ã<EFBFBD>
//<2F><><EFBFBD><EFBFBD>ֵ: 0-<2D>ɹ<EFBFBD><C9B9><EFBFBD><0ʧ<30>ܣ<EFBFBD><DCA3><EFBFBD>ͬ
int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct future* future,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size,
struct tango_cache_meta *meta,
@@ -102,15 +117,15 @@ int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct
enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf,
struct tango_cache_meta *meta,
char *objectkey, size_t keysize);
//<EFBFBD><EFBFBD>ʽ<EFBFBD>ϴ<EFBFBD>
//<2F><>tango_cache_update_start<EFBFBD><EFBFBD><EFBFBD><EFBFBD>NULL<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>tango_cache_ctx_error<EFBFBD>鿴<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><EFBFBD><EFBFBD>CACHE_OUTOF_MEMORY(<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>)
//<2F><>future<72><65>ΪNULL<4C><4C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>֪ͨ<CDA8>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>򲻵<EFBFBD><F2B2BBB5><EFBFBD>
/*<EFBFBD><EFBFBD>ʽ<EFBFBD>ϴ<EFBFBD>API*/
//<2F><><EFBFBD><EFBFBD>ֵ: <EFBFBD><EFBFBD>ΪNULL<EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʧ<EFBFBD>ܣ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>tango_cache_ctx_error<EFBFBD>鿴<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><EFBFBD><EFBFBD>CACHE_OUTOF_MEMORY(<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>)<EFBFBD><EFBFBD>
struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta);
//<2F><><EFBFBD><EFBFBD>ֵ: 0-<2D>ɹ<EFBFBD><C9B9><EFBFBD><0ʧ<30>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD>tango_cache_get_last_error<6F>鿴<EFBFBD><E9BFB4><EFBFBD><EFBFBD><EFBFBD>룻
int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data, size_t size);
int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf);
void tango_cache_update_end(struct tango_cache_ctx *ctx);
//<2F><>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD>keyֵ<79><D6B5><EFBFBD><EFBFBD>CACHE_OBJECT_KEY_HASH_SWITCH=1<><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>ϣʱ<CFA3><CAB1><EFBFBD><EFBFBD>
//<2F><>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD>keyֵ<79><D6B5><EFBFBD><EFBFBD>CACHE_OBJECT_KEY_HASH_SWITCH=1<><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD>URL/<EFBFBD>ļ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϣʱ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>
const char *tango_cache_get_object_key(struct tango_cache_ctx *ctx);
#endif

32
cache/tango_cache_client.cpp vendored
View File

@@ -54,9 +54,14 @@ void tango_cache_get_statistics(const struct tango_cache_instance *instance, str
out->memory_used = instance->statistic.memory_used;
}
struct tango_cache_result *tango_cache_read_result(void *promise_result)
{
return (struct tango_cache_result *)promise_result;
}
const char *tango_cache_get_object_key(struct tango_cache_ctx *ctx)
{
return ctx->file_key;
return ctx->object_key;
}
static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, struct cache_statistics *statistic)
@@ -117,7 +122,11 @@ void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx)
}
easy_string_destroy(&ctx->response);
if(ctx->method == CACHE_REQUEST_PUT)
if(ctx->method == CACHE_REQUEST_GET)
{
easy_string_destroy(&ctx->response_tag);
}
else
{
struct multipart_etag_list *etag;
@@ -208,7 +217,6 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
{
struct tango_cache_ctx *ctx;
char buffer[256]={0};
int other_len;
time_t expires, now, last_modify;
if((u_int64_t)instance->statistic.memory_used >= instance->cache_limit_size)
@@ -227,11 +235,11 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
if(instance->hash_object_key)
{
caculate_sha256(meta->url, strlen(meta->url), buffer, 72);
snprintf(ctx->file_key, 72, "%c%c/%c%c/%s", buffer[0], buffer[1], buffer[2], buffer[3], buffer+4);
snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", buffer[0], buffer[1], buffer[2], buffer[3], buffer+4);
}
else
{
snprintf(ctx->file_key, 72, "%s", meta->url);
snprintf(ctx->object_key, 256, "%s", meta->url);
}
//Expires<65>ֶΣ<D6B6><CEA3><EFBFBD><EFBFBD>ڻ<EFBFBD><DABB><EFBFBD><EFBFBD>ڲ<EFBFBD><DAB2>ж<EFBFBD><D0B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>ʱ
@@ -258,11 +266,11 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
}
}
//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD>GETʱ<54><CAB1>ԭ<EFBFBD><D4AD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
if(meta->other_hdr!=NULL && (other_len = strlen(meta->other_hdr))<1535)
if(meta->usertag_len>0 && meta->usertag_len<=USER_TAG_MAX_LEN)
{
char *p = (char *)malloc((other_len/3 + 1)*4 + 18); //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ռ䣻18=17+1: ͷ<><CDB7>+<2B>ַ<EFBFBD><D6B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
char *p = (char *)malloc((meta->usertag_len/3 + 1)*4 + 18); //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ռ䣻18=17+1: ͷ<><CDB7>+<2B>ַ<EFBFBD><D6B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
memcpy(p, "x-amz-meta-user: ", 17);
Base64_EncodeBlock((unsigned char*)meta->other_hdr, other_len, (unsigned char*)p+17);
Base64_EncodeBlock((const unsigned char*)meta->usertag, meta->usertag_len, (unsigned char*)p+17);
ctx->headers = curl_slist_append(ctx->headers, p);
free(p);
}
@@ -297,7 +305,7 @@ int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct f
}
if(objectkey != NULL)
{
snprintf(objectkey, keysize, "%s", ctx->file_key);
snprintf(objectkey, keysize, "%s", ctx->object_key);
}
return tango_cache_upload_once_start_data(ctx, way, data, size);
@@ -315,7 +323,7 @@ int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct
}
if(objectkey != NULL)
{
snprintf(objectkey, keysize, "%s", ctx->file_key);
snprintf(objectkey, keysize, "%s", ctx->object_key);
}
return tango_cache_upload_once_start_evbuf(ctx, way, evbuf);
@@ -338,11 +346,11 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i
if(instance->hash_object_key)
{
caculate_sha256(meta->url, strlen(meta->url), sha256, 72);
snprintf(ctx->file_key, 72, "%c%c/%c%c/%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
snprintf(ctx->object_key, 256, "%c%c/%c%c/%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
}
else
{
snprintf(ctx->file_key, 72, "%s", meta->url);
snprintf(ctx->object_key, 256, "%s", meta->url);
}
return ctx;

8
cache/tango_cache_client_in.h vendored
View File

@@ -54,7 +54,6 @@ struct tango_cache_instance
time_t relative_ttl; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ч<EFBFBD><D0A7>
u_int64_t cache_limit_size;
long max_cnn_host;
int host_index;
u_int32_t block_len; //<2F><><EFBFBD><EFBFBD>buffercache<68>ڴ<EFBFBD><DAB4>Ļ<EFBFBD><C4BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1>ÿ<EFBFBD><C3BF>update<74><65><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD>ò<EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ӿ<EFBFBD><D3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
u_int32_t upload_block_size; //minio<69>ֶ<EFBFBD><D6B6>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD><EFBFBD>
enum CACHE_ERR_CODE error_code;
@@ -74,10 +73,8 @@ struct tango_cache_ctx
struct curl_slist *headers;
struct future* future;
char error[CURL_ERROR_SIZE];
char file_key[72];
char object_key[256];
char hostport[24]; //<2F><>ͬctxʹ<78><CAB9><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC>IP<49><50><EFBFBD><EFBFBD>֤pipeline˳<65><CBB3><EFBFBD><EFBFBD>
u_int32_t host_index;
bool fail_state;
enum CACHE_REQUEST_METHOD method;
enum CACHE_ERR_CODE error_code;
@@ -89,6 +86,7 @@ struct tango_cache_ctx
};
u_int32_t part_index;
u_int32_t need_hdrs; //<2F><>RESPONSE_HDR_
bool fail_state;
bool close_state; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ùر<C3B9>
long res_code;
time_t max_age;//Get
@@ -101,6 +99,7 @@ struct tango_cache_ctx
char *uploadID;
char *combine_xml;
struct easy_string response;
struct easy_string response_tag;
TAILQ_HEAD(__etag_list_head, multipart_etag_list) cache_head;
struct tango_cache_instance *instance;
@@ -112,6 +111,7 @@ struct curl_socket_data
};
void easy_string_savedata(struct easy_string *estr, const char *data, size_t len);
void easy_string_expand(struct easy_string *estr, size_t to_size);
void easy_string_destroy(struct easy_string *estr);
void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx);

55
cache/tango_cache_transfer.cpp vendored
View File

@@ -93,7 +93,7 @@ static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, v
ctx->upload_offset += send_len;
ctx->instance->statistic.memory_used -= send_len;
return len;
return send_len;
}
//return value: <0:fail; =0: not exec; >0: OK
@@ -110,11 +110,11 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
ctx->upload_offset = 0;
if(full)
{
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
}
else
{
snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ++ctx->part_index, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?partNumber=%d&uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ++ctx->part_index, ctx->uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
}
@@ -129,7 +129,7 @@ static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool ful
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->upload_length);
@@ -179,7 +179,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/%s?uploads", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s?uploads", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -192,7 +192,7 @@ int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
@@ -209,7 +209,7 @@ bool cache_delete_minio_object(struct tango_cache_ctx *ctx)
return false;
}
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -237,7 +237,7 @@ bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx)
return false;
}
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -267,7 +267,7 @@ bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
}
construct_complete_xml(ctx, &ctx->combine_xml, &len);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key, ctx->uploadID);
snprintf(minio_url, 256, "http://%s/%s/%s?uploadId=%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key, ctx->uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
@@ -507,7 +507,7 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM
}
ctx->put_state = PUT_STATE_END;
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL, 1L);
@@ -519,7 +519,7 @@ int tango_cache_upload_once_start_data(struct tango_cache_ctx *ctx, enum PUT_MEM
curl_easy_setopt(ctx->curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
if(way == PUT_MEM_COPY)
{
@@ -670,7 +670,15 @@ static bool check_fresh_header(struct tango_cache_ctx *ctx)
return false;
}
if(ctx->response.buff != NULL)
if(ctx->response_tag.len > 0)
{
result.data_frag = ctx->response_tag.buff;
result.size = ctx->response_tag.len;
result.type = RESULT_TYPE_USERTAG;
promise_success(future_to_promise(ctx->future), &result);
easy_string_destroy(&ctx->response_tag);
}
if(ctx->response.len > 0)
{
result.data_frag = ctx->response.buff;
result.size = ctx->response.len;
@@ -703,7 +711,7 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
char *start=(char *)ptr, *pos_colon, *hdrdata=(char*)ptr;
bool ptr_valid=false;
size_t raw_len = size*count, hdrlen=size*count;
char usrhdr[2048];
char usertag[2048];
if(ctx->fail_state || ctx->get_state==GET_STATE_DELETE)
{
@@ -714,6 +722,8 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
return raw_len;
}
result.type = RESULT_TYPE_HEADER;
if((pos_colon=(char*)memchr(start, ':', raw_len))!=NULL)
{
size_t datalen = pos_colon - start;
@@ -743,10 +753,18 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
case 15:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15))
{
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usrhdr, 2048))>0)
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0)
{
if(ctx->need_hdrs == RESPONSE_HDR_ALL)
{
hdrdata = usrhdr;
ptr_valid = true;
result.type = RESULT_TYPE_USERTAG;
hdrdata = usertag;
}
else
{
easy_string_savedata(&ctx->response_tag, usertag, hdrlen);
}
}
}
break;
@@ -765,10 +783,9 @@ static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count,
{
result.data_frag = hdrdata;
result.size = hdrlen;
result.type = RESULT_TYPE_HEADER;
promise_success(future_to_promise(ctx->future), &result);
}
else
else if(result.type == RESULT_TYPE_HEADER)
{
easy_string_savedata(&ctx->response, hdrdata, hdrlen);
}
@@ -787,7 +804,7 @@ int tango_cache_fetch_start(struct tango_cache_ctx *ctx)
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->file_key);
snprintf(minio_url, 256, "http://%s/%s/%s", ctx->instance->minio_hostlist, ctx->instance->bucketname, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
curl_easy_setopt(ctx->curl, CURLOPT_NOSIGNAL,1L);
@@ -801,7 +818,7 @@ int tango_cache_fetch_start(struct tango_cache_ctx *ctx)
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
//ctx->error="Operation too slow. Less than 1024 bytes/sec transferred the last 3 seconds"
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_TIME, 2L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
curl_easy_setopt(ctx->curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);

19
cache/test_demo/cache_evbase_test.cpp vendored
View File

@@ -33,21 +33,25 @@ struct future_pdata
void get_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = (struct tango_cache_result *)result;
struct tango_cache_result *res = cache_evbase_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
if(res != NULL)
{
if(res->type==RESULT_TYPE_BODY)
switch(res->type)
{
case RESULT_TYPE_BODY:
fwrite(res->data_frag, res->size, 1, pdata->fp);
}
else
{
break;
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
default:break;
}
}
else //<2F><><EFBFBD><EFBFBD>
@@ -135,7 +139,7 @@ int main(int argc, char **argv)
if(argc != 2 && argc!=3)
{
printf("USGAE: %s <PUT/PUTONCE/GET:filename> [get_out_file_index]\n", argv[0]);
printf("USGAE: %s <PUT/PUTONCE/PUTONCEEV/GET:filename> [get_out_file_index]\n", argv[0]);
return -1;
}
if(argc==3)
@@ -164,7 +168,8 @@ int main(int argc, char **argv)
meta.url = filename_in;
meta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype";
meta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip";
meta.other_hdr = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
meta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
meta.usertag_len = strlen(meta.usertag);
p = method;
while(*p=='\r'||*p=='\n') p++;

2
cache/test_demo/pangu_tg_cahce.conf vendored
View File

@@ -15,7 +15,7 @@ CACHE_BLOCK_MAX_SIZE=8192
MAX_USED_MEMORY_SIZE_MB=5120
#上传时Expires头部的过期时间，单位秒，最小60（1分钟）
CACHE_DEFAULT_TTL_SECOND=3600
CACHE_DEFAULT_TTL_SECOND=60
#是否对对象的名称进行哈希，开启哈希有助于提高上传下载的速率
CACHE_OBJECT_KEY_HASH_SWITCH=1

17
cache/test_demo/tango_cache_test.c vendored
View File

@@ -46,21 +46,25 @@ struct future_pdata
void get_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = (struct tango_cache_result *)result;
struct tango_cache_result *res = tango_cache_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
if(res != NULL)
{
if(res->type==RESULT_TYPE_BODY)
switch(res->type)
{
case RESULT_TYPE_BODY:
fwrite(res->data_frag, res->size, 1, pdata->fp);
}
else
{
break;
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
default:break;
}
}
else
@@ -159,7 +163,8 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
meta.url = s;
meta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype";
meta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip";
meta.other_hdr = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
meta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
meta.usertag_len = strlen(meta.usertag);
while(*p=='\r'||*p=='\n')p++;
if(*p=='\0') continue;