gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HEAD object元信息来源选择由宏定义改为配置文件。

Browse Source
This commit is contained in:
zhangchengwei
2018-10-24 12:03:21 +08:00
committed by zhengchao
parent e65f0150b8
commit 268e1b8689
12 changed files with 71 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
cache/src/cache_evbase_client.cpp vendored
View File

@@ -17,9 +17,7 @@
#include "cache_evbase_client.h"
#include "tango_cache_transfer.h"
#include "tango_cache_tools.h"
#ifdef HEAD_OBJECT_FROM_REDIS
#include "tango_cache_redis.h"
#endif
enum CACHE_ASYN_CMD
{
@@ -163,11 +161,14 @@ static void cache_asyn_ioevent_dispatch(struct databuffer *buffer)
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_HEAD:
#ifdef HEAD_OBJECT_FROM_REDIS
tango_cache_head_redis(ctx_asyn->ctx);
#else
tango_cache_fetch_start(ctx_asyn->ctx);
#endif
if(ctx_asyn->instance_asyn->instance->head_meta_source == HEAD_META_FROM_REDIS)
{
tango_cache_head_redis(ctx_asyn->ctx);
}
else
{
tango_cache_fetch_start(ctx_asyn->ctx);
}
cache_asyn_ctx_destroy(ctx_asyn);
break;

54
cache/src/tango_cache_client.cpp vendored
View File

@@ -17,9 +17,7 @@
#include "tango_cache_transfer.h"
#include "tango_cache_tools.h"
#include "tango_cache_xml.h"
#ifdef HEAD_OBJECT_FROM_REDIS
#include "tango_cache_redis.h"
#endif
int TANGO_CACHE_VERSION_20181009=0;
@@ -491,11 +489,15 @@ int tango_cache_head_object(struct tango_cache_instance *instance, struct future
{
return -1;
}
#ifdef HEAD_OBJECT_FROM_REDIS
return tango_cache_head_redis(ctx);
#else
return tango_cache_fetch_start(ctx);
#endif
if(ctx->instance->head_meta_source == HEAD_META_FROM_REDIS)
{
return tango_cache_head_redis(ctx);
}
else
{
return tango_cache_fetch_start(ctx);
}
}
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* f, const char *objkey)
@@ -761,15 +763,18 @@ static int load_local_configure(struct tango_cache_instance *instance, const cha
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_BUCKET_NAME not found.\n", profile_path, section);
return -1;
}
#ifdef HEAD_OBJECT_FROM_REDIS
MESA_load_profile_string_def(profile_path, section, "CACHE_REDIS_KEY", instance->redis_key, 256, instance->bucketname);
if(MESA_load_profile_string_nodef(profile_path, section, "CACHE_REDIS_IP", instance->redis_ip, 256) < 0)
MESA_load_profile_int_def(profile_path, section, "CACHE_HEAD_FROM_SOURCE", &instance->head_meta_source, HEAD_META_FROM_MINIO);
if(instance->head_meta_source == HEAD_META_FROM_REDIS)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_REDIS_IP not found.\n", profile_path, section);
return -1;
MESA_load_profile_string_def(profile_path, section, "CACHE_HEAD_REDIS_KEY", instance->redis_key, 256, instance->bucketname);
if(MESA_load_profile_string_nodef(profile_path, section, "CACHE_HEAD_REDIS_IP", instance->redis_ip, 256) < 0)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_HEAD_REDIS_IP not found.\n", profile_path, section);
return -1;
}
MESA_load_profile_int_def(profile_path, section, "CACHE_HEAD_REDIS_PORT", &instance->redis_port, 6379);
}
MESA_load_profile_int_def(profile_path, section, "CACHE_REDIS_PORT", &instance->redis_port, 6379);
#endif
MESA_load_profile_uint_def(profile_path, section, "CACHE_OBJECT_KEY_HASH_SWITCH", &instance->hash_object_key, 1);
MESA_load_profile_uint_def(profile_path, section, "MINIO_LISTEN_PORT", &instance->minio_port, 9000);
if(MESA_load_profile_string_nodef(profile_path, section, "MINIO_IP_LIST", instance->minio_iplist, 4096) < 0)
@@ -829,18 +834,19 @@ struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,
curl_multi_setopt(instance->multi_hd, CURLMOPT_TIMERFUNCTION, curl_timer_function_cb);
curl_multi_setopt(instance->multi_hd, CURLMOPT_TIMERDATA, instance);
#ifdef HEAD_OBJECT_FROM_REDIS
if(redis_asyn_connect_init(instance, instance->redis_ip, instance->redis_port))
if(instance->head_meta_source == HEAD_META_FROM_REDIS)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s:%u failed.", instance->redis_ip, instance->redis_port);
free(instance);
return NULL;
if(redis_asyn_connect_init(instance))
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s:%u failed.", instance->redis_ip, instance->redis_port);
free(instance);
return NULL;
}
else
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s:%u success.", instance->redis_ip, instance->redis_port);
}
}
else
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s:%u success.", instance->redis_ip, instance->redis_port);
}
#endif
evtimer_assign(&instance->timer_event, evbase, libevent_timer_event_cb, instance);
return instance;
}

10
cache/src/tango_cache_client_in.h vendored
View File

@@ -6,9 +6,7 @@
#include <event2/event.h>
#include <event.h>
#ifdef HEAD_OBJECT_FROM_REDIS
#include <hiredis/async.h>
#endif
#include <MESA/wiredLB.h>
#include "tango_cache_client.h"
@@ -17,6 +15,9 @@
#define RESPONSE_HDR_LAST_MOD 2
#define RESPONSE_HDR_ALL 3
#define HEAD_META_FROM_MINIO 1
#define HEAD_META_FROM_REDIS 2
enum CACHE_REQUEST_METHOD
{
CACHE_REQUEST_GET=0,
@@ -71,13 +72,14 @@ struct tango_cache_instance
long max_cnn_host;
u_int32_t upload_block_size; //minio<69>ֶ<EFBFBD><D6B6>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD><EFBFBD>
enum CACHE_ERR_CODE error_code;
#ifdef HEAD_OBJECT_FROM_REDIS
int head_meta_source; //<2F><><EFBFBD>Դ<EFBFBD>MINIO<49><4F>REDIS<49><53>ȡԪ<C8A1><D4AA>Ϣ
//Ԫ<><D4AA>Ϣ<EFBFBD><CFA2>ȡ<EFBFBD><C8A1>ʽRedis
redisAsyncContext *redis_ac;
char redis_key[256];
char redis_ip[128];
int redis_port;
int redis_connecting;
#endif
};
struct multipart_etag_list

10
cache/src/tango_cache_redis.cpp vendored
View File

@@ -70,9 +70,9 @@ void redis_asyn_connect_cb(const struct redisAsyncContext *ac, int status)
}
}
int redis_asyn_connect_init(struct tango_cache_instance *instance, const char *redisip, int redis_port)
int redis_asyn_connect_init(struct tango_cache_instance *instance)
{
instance->redis_ac = redisAsyncConnect(redisip, redis_port);
instance->redis_ac = redisAsyncConnect(instance->redis_ip, instance->redis_port);
if(instance->redis_ac == NULL)
{
return -1;
@@ -226,10 +226,10 @@ int tango_cache_head_redis(struct tango_cache_ctx *ctx)
case CACHE_REDIS_CONNECTED:
ret = redisAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx, "HGET %s %s/%s",
ctx->instance->redis_key, ctx->instance->bucketname, ctx->object_key);
if(ret < 0)
if(ret != REDIS_OK)
{
//redisAsyncDisconnect(ctx->instance->redis_ac);
redis_asyn_connect_init(ctx->instance, ctx->instance->redis_ip, ctx->instance->redis_port);
redis_asyn_connect_init(ctx->instance);
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx));
tango_cache_ctx_destroy(ctx);
@@ -237,7 +237,7 @@ int tango_cache_head_redis(struct tango_cache_ctx *ctx)
break;
case CACHE_REDIS_DISCONNECTED:
case CACHE_REDIS_CONNECT_IDLE:
redis_asyn_connect_init(ctx->instance, ctx->instance->redis_ip, ctx->instance->redis_port);
redis_asyn_connect_init(ctx->instance);
case CACHE_REDIS_CONNECTING:
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx));

2
cache/src/tango_cache_redis.h vendored
View File

@@ -7,7 +7,7 @@
#include "tango_cache_client_in.h"
int tango_cache_head_redis(struct tango_cache_ctx *ctx);
int redis_asyn_connect_init(struct tango_cache_instance *instance, const char *redisip, int redis_port);
int redis_asyn_connect_init(struct tango_cache_instance *instance);
#endif

9
cache/test/Makefile vendored
View File

@@ -1,11 +1,12 @@
CCC=g++
INC_PATH=-I../src/include -I../src/include/libevent2
INC_PATH=-I../include -I../include/libevent2
CFLAGS=-Wall -g $(INC_PATH)
LIBS = -lMESA_handle_logger -lMESA_htable -lMESA_prof_load -lWiredLB
LIBS += -lssl -lcrypto -lcurl -levent -lxml2
LIBS += ../src/pangu_tango_cache.a
LIBS = -lMESA_handle_logger -lMESA_prof_load -lWiredLB
LIBS += -lssl -lcrypto
LIBS += ../lib/libtango_cache_client.a
LIBS += ./lib/libcurl.a ./lib/libevent.a ./lib/libxml2.a ./lib/libhiredis.a ./lib/libhiredis.a
OBJS = tango_cache_test.o
OBJS_EVBASE=cache_evbase_test.o

BIN
cache/test/lib/libcurl.a vendored Normal file
View File

Binary file not shown.

BIN
cache/test/lib/libevent.a vendored Normal file
View File

Binary file not shown.

BIN
cache/test/lib/libhiredis.a vendored Normal file
View File

Binary file not shown.

BIN
cache/test/lib/libxml2.a vendored Normal file
View File

Binary file not shown.

26
cache/test/pangu_tg_cahce.conf vendored
View File

@@ -1,24 +1,30 @@
[TANGO_CACHE]
#Address of MINIO Servers
#MINIO IP地址，目前只支持一个
MINIO_IP_LIST=192.168.10.61-64;
MINIO_LISTEN_PORT=9000
#每个域名最多开启的链接数
MAX_CONNECTION_PER_HOST=10
CACHE_BUCKET_NAME=openbucket
#Upload failed when exceed max memory constraintt
#bucket的名称
CACHE_BUCKET_NAME=images
#缓存最大占用的内存空间大小，超出空间时上传失败
MAX_USED_MEMORY_SIZE_MB=5120
#Expire second of Upload header, 60s minimum.
#上传时Expires头部的过期时间，单位秒，最小60（1分钟）
CACHE_DEFAULT_TTL_SECOND=3600
#是否对对象的名称进行哈希，开启哈希有助于提高上传下载的速率
CACHE_OBJECT_KEY_HASH_SWITCH=1
#Hash object name to speedup query.
CACHE_OBJECT_KEY_HASH_SWITCH=0
#HEAD元信息的来源，1-MINIO，2-REDIS
CACHE_HEAD_FROM_SOURCE=1
#使用Redis作为元信息获取源
CACHE_HEAD_REDIS_KEY=MINIO_EVENTS_INFO
CACHE_HEAD_REDIS_IP=192.168.10.63
CACHE_HEAD_REDIS_PORT=6379
#For WIRED LOAD BALANCER
#WIRED LOAD BALANCER配置
#WIREDLB_OVERRIDE=1
#WIREDLB_TOPIC=
#WIREDLB_GROUP=
#WIREDLB_DATACENTER=
#WIREDLB_HEALTH_PORT=52100

BIN
vendor/hiredis-master.zip vendored
View File

Binary file not shown.