diff --git a/cache/src/cache_evbase_client.cpp b/cache/src/cache_evbase_client.cpp index 6cd4ea6..4ce8799 100644 --- a/cache/src/cache_evbase_client.cpp +++ b/cache/src/cache_evbase_client.cpp @@ -17,9 +17,7 @@ #include "cache_evbase_client.h" #include "tango_cache_transfer.h" #include "tango_cache_tools.h" -#ifdef HEAD_OBJECT_FROM_REDIS #include "tango_cache_redis.h" -#endif enum CACHE_ASYN_CMD { @@ -163,11 +161,14 @@ static void cache_asyn_ioevent_dispatch(struct databuffer *buffer) cache_asyn_ctx_destroy(ctx_asyn); break; case CACHE_ASYN_HEAD: -#ifdef HEAD_OBJECT_FROM_REDIS - tango_cache_head_redis(ctx_asyn->ctx); -#else - tango_cache_fetch_start(ctx_asyn->ctx); -#endif + if(ctx_asyn->instance_asyn->instance->head_meta_source == HEAD_META_FROM_REDIS) + { + tango_cache_head_redis(ctx_asyn->ctx); + } + else + { + tango_cache_fetch_start(ctx_asyn->ctx); + } cache_asyn_ctx_destroy(ctx_asyn); break; diff --git a/cache/src/tango_cache_client.cpp b/cache/src/tango_cache_client.cpp index 1575437..0efec36 100644 --- a/cache/src/tango_cache_client.cpp +++ b/cache/src/tango_cache_client.cpp @@ -17,9 +17,7 @@ #include "tango_cache_transfer.h" #include "tango_cache_tools.h" #include "tango_cache_xml.h" -#ifdef HEAD_OBJECT_FROM_REDIS #include "tango_cache_redis.h" -#endif int TANGO_CACHE_VERSION_20181009=0; @@ -491,11 +489,15 @@ int tango_cache_head_object(struct tango_cache_instance *instance, struct future { return -1; } -#ifdef HEAD_OBJECT_FROM_REDIS - return tango_cache_head_redis(ctx); -#else - return tango_cache_fetch_start(ctx); -#endif + + if(ctx->instance->head_meta_source == HEAD_META_FROM_REDIS) + { + return tango_cache_head_redis(ctx); + } + else + { + return tango_cache_fetch_start(ctx); + } } struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* f, const char *objkey) @@ -761,15 +763,18 @@ static int load_local_configure(struct tango_cache_instance *instance, const cha MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_BUCKET_NAME not found.\n", profile_path, section); return -1; } -#ifdef HEAD_OBJECT_FROM_REDIS - MESA_load_profile_string_def(profile_path, section, "CACHE_REDIS_KEY", instance->redis_key, 256, instance->bucketname); - if(MESA_load_profile_string_nodef(profile_path, section, "CACHE_REDIS_IP", instance->redis_ip, 256) < 0) + + MESA_load_profile_int_def(profile_path, section, "CACHE_HEAD_FROM_SOURCE", &instance->head_meta_source, HEAD_META_FROM_MINIO); + if(instance->head_meta_source == HEAD_META_FROM_REDIS) { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_REDIS_IP not found.\n", profile_path, section); - return -1; + MESA_load_profile_string_def(profile_path, section, "CACHE_HEAD_REDIS_KEY", instance->redis_key, 256, instance->bucketname); + if(MESA_load_profile_string_nodef(profile_path, section, "CACHE_HEAD_REDIS_IP", instance->redis_ip, 256) < 0) + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_HEAD_REDIS_IP not found.\n", profile_path, section); + return -1; + } + MESA_load_profile_int_def(profile_path, section, "CACHE_HEAD_REDIS_PORT", &instance->redis_port, 6379); } - MESA_load_profile_int_def(profile_path, section, "CACHE_REDIS_PORT", &instance->redis_port, 6379); -#endif MESA_load_profile_uint_def(profile_path, section, "CACHE_OBJECT_KEY_HASH_SWITCH", &instance->hash_object_key, 1); MESA_load_profile_uint_def(profile_path, section, "MINIO_LISTEN_PORT", &instance->minio_port, 9000); if(MESA_load_profile_string_nodef(profile_path, section, "MINIO_IP_LIST", instance->minio_iplist, 4096) < 0) @@ -829,18 +834,19 @@ struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase, curl_multi_setopt(instance->multi_hd, CURLMOPT_TIMERFUNCTION, curl_timer_function_cb); curl_multi_setopt(instance->multi_hd, CURLMOPT_TIMERDATA, instance); -#ifdef HEAD_OBJECT_FROM_REDIS - if(redis_asyn_connect_init(instance, instance->redis_ip, instance->redis_port)) + if(instance->head_meta_source == HEAD_META_FROM_REDIS) { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s:%u failed.", instance->redis_ip, instance->redis_port); - free(instance); - return NULL; + if(redis_asyn_connect_init(instance)) + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s:%u failed.", instance->redis_ip, instance->redis_port); + free(instance); + return NULL; + } + else + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s:%u success.", instance->redis_ip, instance->redis_port); + } } - else - { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s:%u success.", instance->redis_ip, instance->redis_port); - } -#endif evtimer_assign(&instance->timer_event, evbase, libevent_timer_event_cb, instance); return instance; } diff --git a/cache/src/tango_cache_client_in.h b/cache/src/tango_cache_client_in.h index aa67ec8..bd82dbc 100644 --- a/cache/src/tango_cache_client_in.h +++ b/cache/src/tango_cache_client_in.h @@ -6,9 +6,7 @@ #include #include -#ifdef HEAD_OBJECT_FROM_REDIS #include -#endif #include #include "tango_cache_client.h" @@ -17,6 +15,9 @@ #define RESPONSE_HDR_LAST_MOD 2 #define RESPONSE_HDR_ALL 3 +#define HEAD_META_FROM_MINIO 1 +#define HEAD_META_FROM_REDIS 2 + enum CACHE_REQUEST_METHOD { CACHE_REQUEST_GET=0, @@ -71,13 +72,14 @@ struct tango_cache_instance long max_cnn_host; u_int32_t upload_block_size; //minio分段上传块的最小长度 enum CACHE_ERR_CODE error_code; -#ifdef HEAD_OBJECT_FROM_REDIS + + int head_meta_source; //可以从MINIO或REDIS获取元信息 + //元信息获取方式Redis redisAsyncContext *redis_ac; char redis_key[256]; char redis_ip[128]; int redis_port; int redis_connecting; -#endif }; struct multipart_etag_list diff --git a/cache/src/tango_cache_redis.cpp b/cache/src/tango_cache_redis.cpp index b66f2eb..8557110 100644 --- a/cache/src/tango_cache_redis.cpp +++ b/cache/src/tango_cache_redis.cpp @@ -70,9 +70,9 @@ void redis_asyn_connect_cb(const struct redisAsyncContext *ac, int status) } } -int redis_asyn_connect_init(struct tango_cache_instance *instance, const char *redisip, int redis_port) +int redis_asyn_connect_init(struct tango_cache_instance *instance) { - instance->redis_ac = redisAsyncConnect(redisip, redis_port); + instance->redis_ac = redisAsyncConnect(instance->redis_ip, instance->redis_port); if(instance->redis_ac == NULL) { return -1; @@ -226,10 +226,10 @@ int tango_cache_head_redis(struct tango_cache_ctx *ctx) case CACHE_REDIS_CONNECTED: ret = redisAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx, "HGET %s %s/%s", ctx->instance->redis_key, ctx->instance->bucketname, ctx->object_key); - if(ret < 0) + if(ret != REDIS_OK) { //redisAsyncDisconnect(ctx->instance->redis_ac); - redis_asyn_connect_init(ctx->instance, ctx->instance->redis_ip, ctx->instance->redis_port); + redis_asyn_connect_init(ctx->instance); tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); tango_cache_ctx_destroy(ctx); @@ -237,7 +237,7 @@ int tango_cache_head_redis(struct tango_cache_ctx *ctx) break; case CACHE_REDIS_DISCONNECTED: case CACHE_REDIS_CONNECT_IDLE: - redis_asyn_connect_init(ctx->instance, ctx->instance->redis_ip, ctx->instance->redis_port); + redis_asyn_connect_init(ctx->instance); case CACHE_REDIS_CONNECTING: tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); promise_failed(future_to_promise(ctx->future), FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); diff --git a/cache/src/tango_cache_redis.h b/cache/src/tango_cache_redis.h index de0acae..8126025 100644 --- a/cache/src/tango_cache_redis.h +++ b/cache/src/tango_cache_redis.h @@ -7,7 +7,7 @@ #include "tango_cache_client_in.h" int tango_cache_head_redis(struct tango_cache_ctx *ctx); -int redis_asyn_connect_init(struct tango_cache_instance *instance, const char *redisip, int redis_port); +int redis_asyn_connect_init(struct tango_cache_instance *instance); #endif diff --git a/cache/test/Makefile b/cache/test/Makefile index 509c8c3..07f89f6 100644 --- a/cache/test/Makefile +++ b/cache/test/Makefile @@ -1,11 +1,12 @@ CCC=g++ -INC_PATH=-I../src/include -I../src/include/libevent2 +INC_PATH=-I../include -I../include/libevent2 CFLAGS=-Wall -g $(INC_PATH) -LIBS = -lMESA_handle_logger -lMESA_htable -lMESA_prof_load -lWiredLB -LIBS += -lssl -lcrypto -lcurl -levent -lxml2 -LIBS += ../src/pangu_tango_cache.a +LIBS = -lMESA_handle_logger -lMESA_prof_load -lWiredLB +LIBS += -lssl -lcrypto +LIBS += ../lib/libtango_cache_client.a +LIBS += ./lib/libcurl.a ./lib/libevent.a ./lib/libxml2.a ./lib/libhiredis.a ./lib/libhiredis.a OBJS = tango_cache_test.o OBJS_EVBASE=cache_evbase_test.o diff --git a/cache/test/lib/libcurl.a b/cache/test/lib/libcurl.a new file mode 100644 index 0000000..7b0c467 Binary files /dev/null and b/cache/test/lib/libcurl.a differ diff --git a/cache/test/lib/libevent.a b/cache/test/lib/libevent.a new file mode 100644 index 0000000..5495d20 Binary files /dev/null and b/cache/test/lib/libevent.a differ diff --git a/cache/test/lib/libhiredis.a b/cache/test/lib/libhiredis.a new file mode 100644 index 0000000..26434df Binary files /dev/null and b/cache/test/lib/libhiredis.a differ diff --git a/cache/test/lib/libxml2.a b/cache/test/lib/libxml2.a new file mode 100644 index 0000000..f39781b Binary files /dev/null and b/cache/test/lib/libxml2.a differ diff --git a/cache/test/pangu_tg_cahce.conf b/cache/test/pangu_tg_cahce.conf index a193059..53cca1d 100644 --- a/cache/test/pangu_tg_cahce.conf +++ b/cache/test/pangu_tg_cahce.conf @@ -1,24 +1,30 @@ [TANGO_CACHE] -#Address of MINIO Servers +#MINIO IP鍦板潃锛岀洰鍓嶅彧鏀寔涓涓 MINIO_IP_LIST=192.168.10.61-64; MINIO_LISTEN_PORT=9000 - +#姣忎釜鍩熷悕鏈澶氬紑鍚殑閾炬帴鏁 MAX_CONNECTION_PER_HOST=10 -CACHE_BUCKET_NAME=openbucket - -#Upload failed when exceed max memory constraintt +#bucket鐨勫悕绉 +CACHE_BUCKET_NAME=images +#缂撳瓨鏈澶у崰鐢ㄧ殑鍐呭瓨绌洪棿澶у皬锛岃秴鍑虹┖闂存椂涓婁紶澶辫触 MAX_USED_MEMORY_SIZE_MB=5120 - -#Expire second of Upload header, 60s minimum. +#涓婁紶鏃禘xpires澶撮儴鐨勮繃鏈熸椂闂达紝鍗曚綅绉掞紝鏈灏60锛1鍒嗛挓锛 CACHE_DEFAULT_TTL_SECOND=3600 +#鏄惁瀵瑰璞＄殑鍚嶇О杩涜鍝堝笇锛屽紑鍚搱甯屾湁鍔╀簬鎻愰珮涓婁紶涓嬭浇鐨勯熺巼 +CACHE_OBJECT_KEY_HASH_SWITCH=1 -#Hash object name to speedup query. -CACHE_OBJECT_KEY_HASH_SWITCH=0 +#HEAD鍏冧俊鎭殑鏉ユ簮锛1-MINIO锛2-REDIS +CACHE_HEAD_FROM_SOURCE=1 +#浣跨敤Redis浣滀负鍏冧俊鎭幏鍙栨簮 +CACHE_HEAD_REDIS_KEY=MINIO_EVENTS_INFO +CACHE_HEAD_REDIS_IP=192.168.10.63 +CACHE_HEAD_REDIS_PORT=6379 -#For WIRED LOAD BALANCER +#WIRED LOAD BALANCER閰嶇疆 #WIREDLB_OVERRIDE=1 #WIREDLB_TOPIC= #WIREDLB_GROUP= #WIREDLB_DATACENTER= +#WIREDLB_HEALTH_PORT=52100 diff --git a/vendor/hiredis-master.zip b/vendor/hiredis-master.zip index c522910..509f420 100644 Binary files a/vendor/hiredis-master.zip and b/vendor/hiredis-master.zip differ