gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复内存统计BUG；添加全局初始化API；

Browse Source
This commit is contained in:
zhangchengwei
2018-09-30 19:20:06 +08:00
committed by zhengchao
parent d47599a45d
commit 664911cc23
8 changed files with 414 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
cache/tango_cache_client.cpp vendored
View File

@@ -18,7 +18,7 @@
int TANGO_CACHE_VERSION_20180925=0;
void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_t size)
static void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_t size)
{
SHA256_CTX c;
unsigned char sha256[128];
@@ -33,7 +33,7 @@ void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_
}
}
int wired_load_balancer_lookup(WLB_handle_t wiredlb, const char *key, int keylen, char *host, size_t hostsize)
static int wired_load_balancer_lookup(WLB_handle_t wiredlb, const char *key, int keylen, char *host, size_t hostsize)
{
struct WLB_consumer_t chosen;
@@ -81,7 +81,7 @@ void tango_cache_get_object_path(const struct tango_cache_ctx *ctx, char *path,
snprintf(path, pathsize, "http://%s/%s/%s", ctx->hostaddr, ctx->instance->bucketname, ctx->object_key);
}
static inline void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, enum CACHE_ERR_CODE error_code, struct cache_statistics *statistic)
static void update_statistics(enum CACHE_REQUEST_METHOD method, bool fail_state, enum CACHE_ERR_CODE error_code, struct cache_statistics *statistic)
{
switch(method)
{
@@ -166,7 +166,11 @@ void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx)
if(ctx->put.uploadID != NULL) free(ctx->put.uploadID);
if(ctx->put.combine_xml != NULL) free(ctx->put.combine_xml);
if(ctx->headers != NULL) curl_slist_free_all(ctx->headers);
if(ctx->put.evbuf!=NULL) evbuffer_free(ctx->put.evbuf);
if(ctx->put.evbuf!=NULL)
{
ctx->instance->statistic.memory_used -= evbuffer_get_length(ctx->put.evbuf);
evbuffer_free(ctx->put.evbuf);
}
TAILQ_FOREACH(etag, &ctx->put.etag_head, node)
{
TAILQ_REMOVE(&ctx->put.etag_head, etag, node);
@@ -256,6 +260,7 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
if((u_int64_t)instance->statistic.memory_used >= instance->cache_limit_size)
{
instance->error_code = CACHE_OUTOF_MEMORY;
instance->statistic.totaldrop_num += 1;
return NULL;
}
@@ -276,6 +281,7 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
if(wired_load_balancer_lookup(instance->wiredlb, meta->url, strlen(meta->url), ctx->hostaddr, 48))
{
instance->error_code = CACHE_ERR_WIREDLB;
instance->statistic.totaldrop_num += 1;
free(ctx);
return NULL;
}
@@ -307,6 +313,7 @@ struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *
{
ctx->headers = curl_slist_append(ctx->headers, "Content-Type:");
}
ctx->headers = curl_slist_append(ctx->headers, "Expect:");
//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD>GETʱ<54><CAB1>ԭ<EFBFBD><D4AD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
if(meta->usertag_len>0 && meta->usertag_len<=USER_TAG_MAX_LEN)
{
@@ -397,6 +404,7 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i
if(wired_load_balancer_lookup(instance->wiredlb, meta->url, strlen(meta->url), ctx->hostaddr, 48))
{
instance->error_code = CACHE_ERR_WIREDLB;
instance->statistic.totaldrop_num += 1;
free(ctx);
return NULL;
}
@@ -405,7 +413,14 @@ struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *i
int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta)
{
return tango_cache_fetch_start(tango_cache_fetch_prepare(instance, future, meta));
struct tango_cache_ctx *ctx;
ctx = tango_cache_fetch_prepare(instance, future, meta);
if(ctx == NULL)
{
return -1;
}
return tango_cache_fetch_start(ctx);
}
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* future, const char *objkey)
@@ -430,6 +445,7 @@ struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *
if(wired_load_balancer_lookup(instance->wiredlb, objkey, strlen(objkey), ctx->hostaddr, 48))
{
instance->error_code = CACHE_ERR_WIREDLB;
instance->statistic.totaldrop_num += 1;
free(ctx);
return NULL;
}
@@ -438,7 +454,14 @@ struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *
int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* future, const char *objkey)
{
return (cache_delete_minio_object(tango_cache_delete_prepare(instance, future, objkey))==1)?0:-1;
struct tango_cache_ctx *ctx;
ctx = tango_cache_delete_prepare(instance, future, objkey);
if(ctx == NULL)
{
return -1;
}
return (cache_delete_minio_object(ctx)==1)?0:-1;
}
static void check_multi_info(CURLM *multi)
@@ -579,6 +602,29 @@ static int curl_timer_function_cb(CURLM *multi, long timeout_ms, void *userp)
return 0; //0-success; -1-error
}
static int wired_load_balancer_init(struct tango_cache_instance *instance)
{
instance->wiredlb = wiredLB_create(instance->wiredlb_topic, instance->wiredlb_group, WLB_PRODUCER);
if(instance->wiredlb == NULL)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "wiredLB_create failed.\n");
return -1;
}
wiredLB_set_opt(instance->wiredlb, WLB_OPT_ENABLE_OVERRIDE, &instance->wiredlb_override, sizeof(instance->wiredlb_override));
wiredLB_set_opt(instance->wiredlb, WLB_PROD_OPT_DATACENTER, instance->wiredlb_datacenter, strlen(instance->wiredlb_datacenter)+1);
if(instance->wiredlb_override)
{
wiredLB_set_opt(instance->wiredlb, WLB_PROD_OPT_OVERRIDE_PRIMARY_IP, instance->minio_iplist, strlen(instance->minio_iplist)+1);
wiredLB_set_opt(instance->wiredlb, WLB_PROD_OPT_OVERRIDE_DATAPORT, &instance->minio_port, sizeof(instance->minio_port));
}
if(wiredLB_init(instance->wiredlb) < 0)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "wiredLB_init failed.\n");
return -1;
}
return 0;
}
static int load_local_configure(struct tango_cache_instance *instance, const char* profile_path, const char* section)
{
u_int32_t intval;
@@ -619,29 +665,6 @@ static int load_local_configure(struct tango_cache_instance *instance, const cha
return 0;
}
int wired_load_balancer_init(struct tango_cache_instance *instance)
{
instance->wiredlb = wiredLB_create(instance->wiredlb_topic, instance->wiredlb_group, WLB_PRODUCER);
if(instance->wiredlb == NULL)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "wiredLB_create failed.\n");
return -1;
}
wiredLB_set_opt(instance->wiredlb, WLB_OPT_ENABLE_OVERRIDE, &instance->wiredlb_override, sizeof(instance->wiredlb_override));
wiredLB_set_opt(instance->wiredlb, WLB_PROD_OPT_DATACENTER, instance->wiredlb_datacenter, strlen(instance->wiredlb_datacenter)+1);
if(instance->wiredlb_override)
{
wiredLB_set_opt(instance->wiredlb, WLB_PROD_OPT_OVERRIDE_PRIMARY_IP, instance->minio_iplist, strlen(instance->minio_iplist)+1);
wiredLB_set_opt(instance->wiredlb, WLB_PROD_OPT_OVERRIDE_DATAPORT, &instance->minio_port, sizeof(instance->minio_port));
}
if(wiredLB_init(instance->wiredlb) < 0)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "wiredLB_init failed.\n");
return -1;
}
return 0;
}
struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,const char* profile_path, const char* section, void *runtimelog)
{
struct tango_cache_instance *instance;
@@ -664,7 +687,7 @@ struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,
instance->multi_hd = curl_multi_init();
instance->runtime_log = runtimelog;
curl_multi_setopt(instance->multi_hd, CURLMOPT_PIPELINING, CURLPIPE_HTTP1);
curl_multi_setopt(instance->multi_hd, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX);
curl_multi_setopt(instance->multi_hd, CURLMOPT_MAX_HOST_CONNECTIONS, instance->max_cnn_host);
curl_multi_setopt(instance->multi_hd, CURLMOPT_SOCKETFUNCTION, curl_socket_function_cb);
curl_multi_setopt(instance->multi_hd, CURLMOPT_SOCKETDATA, instance); //curl_socket_function_cb *userp
@@ -676,3 +699,8 @@ struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,
return instance;
}
void tango_cache_global_init(void)
{
curl_global_init(CURL_GLOBAL_NOTHING);
}