gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加集群版redis作为元信息和对象缓存，去除Minio事件通知的redis元信息获取方式。

Browse Source
This commit is contained in:
zhangchengwei
2018-12-14 15:07:09 +08:00
committed by zhengchao
parent 08ae82932a
commit d24c57ce85
32 changed files with 1561 additions and 909 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
cache/src/tango_cache_client_in.h vendored
View File

@@ -6,7 +6,9 @@
#include <event2/event.h>
#include <event.h>
#include <hiredis/async.h>
#include <hiredis-vip/async.h>
#include <hiredis-vip/hircluster.h>
#include <cjson/cJSON.h>
#include <MESA/wiredLB.h>
#include "tango_cache_client.h"
@@ -15,8 +17,9 @@
#define RESPONSE_HDR_LAST_MOD 2
#define RESPONSE_HDR_ALL 3
#define HEAD_META_FROM_MINIO 1
#define HEAD_META_FROM_REDIS 2
#define CACHE_ALL_MINIO 0 //Ԫ<><D4AA>Ϣ<EFBFBD>Ͷ<EFBFBD><CDB6>󶼴<EFBFBD><F3B6BCB4><EFBFBD>MINIO
#define CACHE_META_REDIS 1 //Ԫ<><D4AA>Ϣ<EFBFBD><CFA2>REDIS<49><53><EFBFBD><EFBFBD><EFBFBD><EFBFBD>MINIO
#define CACHE_SMALL_REDIS 2 //Ԫ<><D4AA>Ϣ<EFBFBD><CFA2>С<EFBFBD>ļ<EFBFBD><C4BC><EFBFBD>REDIS<49><53><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD>MINIO
enum CACHE_REQUEST_METHOD
{
@@ -31,6 +34,9 @@ enum GET_OBJECT_STATE
{
GET_STATE_START=0,
GET_STATE_DELETE,
GET_STATE_REDIS_META,
GET_STATE_REDIS_ALL,
GET_STATE_REDIS_TRY,
GET_STATE_END,
};
@@ -39,8 +45,10 @@ enum PUT_OBJECT_STATE
PUT_STATE_START=0,
PUT_STATE_WAIT_START,
PUT_STATE_PART,
PUT_STATE_END,
PUT_STATE_CANCEL,
PUT_STATE_REDIS_META,
PUT_STATE_REDIS_ALL,
PUT_STATE_END,
};
struct easy_string
@@ -52,35 +60,33 @@ struct easy_string
struct wiredlb_parameter
{
char wiredlb_topic[64];
char wiredlb_datacenter[64];
char wiredlb_group[64];
char mainip[64]; //Ĭ<EFBFBD>Ϸ<EFBFBD><EFBFBD>ʵ<EFBFBD>redis<EFBFBD><EFBFBD>ַ
char iplist[4096];//minio: minio<69>б<EFBFBD><D0B1><EFBFBD>redis: mainip<69><70><EFBFBD>˺󣬿<CBBA>ѡ<EFBFBD><D1A1><EFBFBD>б<EFBFBD><D0B1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>mainip
char iplist[4096];//minio<EFBFBD>б<EFBFBD>
WLB_handle_t wiredlb;
u_int32_t wiredlb_override;
u_int32_t port;
short wiredlb_ha_port;
WLB_handle_t wiredlb;
};
struct tango_cache_parameter
{
char bucketname[256];
char redis_key[256];
long max_cnn_host;
long maximum_host_cnns;
long transfer_timeout;//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
long max_pipeline_num;
u_int64_t cache_limit_size;
u_int32_t max_session_num;
long maximum_pipelines;
u_int64_t maximum_used_mem;
u_int32_t maximum_sessions;
u_int32_t upload_block_size; //minio<69>ֶ<EFBFBD><D6B6>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD><EFBFBD>
time_t relative_ttl; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ч<EFBFBD><D0A7>
u_int32_t hash_object_key;
//wiredlb
int head_meta_source; //<2F><><EFBFBD>Դ<EFBFBD>MINIO<49><4F>REDIS<49><53>ȡԪ<C8A1><D4AA>Ϣ
u_int32_t wiredlb_override;
char wiredlb_topic[64];
char wiredlb_datacenter[64];
int object_store_way; //<2F><>ȡobject<63><74>Ϣ<EFBFBD>ķ<EFBFBD>ʽ
struct wiredlb_parameter minio;
struct wiredlb_parameter redis;
char redisaddrs[4096];
u_int32_t redis_object_maxsize;//С<>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>redisʱ<73><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С
};
struct tango_cache_instance
@@ -90,11 +96,9 @@ struct tango_cache_instance
CURLM *multi_hd;
enum CACHE_ERR_CODE error_code;
//Ԫ<><D4AA>Ϣ<EFBFBD><CFA2>ȡ<EFBFBD><C8A1>ʽRedis
int redis_connecting;
redisAsyncContext *redis_ac;
char current_redisip[64];
struct event timer_redis;
redisClusterAsyncContext *redis_ac;
char redisaddr[128];
const struct tango_cache_parameter *param;
void *runtime_log;
@@ -108,6 +112,8 @@ struct multipart_etag_list
TAILQ_ENTRY(multipart_etag_list) node;
};
typedef void (redisRedirectMinioCallback)(struct tango_cache_ctx *ctx);
struct cache_ctx_data_get
{
time_t max_age;
@@ -118,6 +124,7 @@ struct cache_ctx_data_get
enum GET_OBJECT_STATE state;
struct easy_string response_tag;
struct tango_cache_result result;
redisRedirectMinioCallback *redis_redirect_minio_cb;
};
struct cache_ctx_data_put
@@ -128,9 +135,12 @@ struct cache_ctx_data_put
char *uploadID;
char *combine_xml;
TAILQ_HEAD(__etag_list_head, multipart_etag_list) etag_head;
cJSON *object_meta;
enum PUT_OBJECT_STATE state;
u_int32_t part_index; //<2F><>RESPONSE_HDR_
u_int32_t object_ttl;
bool close_state; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ùر<C3B9>
size_t object_size;
};
struct cache_ctx_multi_delete
@@ -153,6 +163,7 @@ struct tango_cache_ctx
struct easy_string response;
bool fail_state;
enum OBJECT_LOCATION locate; //<2F>ɳ<EFBFBD><C9B3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>϶<EFBFBD><CFB6><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD><CEBB>
long res_code;
union{
@@ -177,9 +188,19 @@ void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx, bool callback=true);
void tango_cache_set_fail_state(struct tango_cache_ctx *ctx, enum CACHE_ERR_CODE error_code);
const char *tango_cache_get_errstring(const struct tango_cache_ctx *ctx);
struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_put *meta);
struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance, enum CACHE_REQUEST_METHOD method, struct future* f, struct tango_cache_meta_get *meta);
bool sessions_exceeds_limit(struct tango_cache_instance *instance, enum OBJECT_LOCATION where_to_get);
struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance,
struct future* f, struct tango_cache_meta_put *meta, enum OBJECT_LOCATION maybe_loc);
struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance,
enum CACHE_REQUEST_METHOD method, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get);
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* f, const char *objkey);
enum OBJECT_LOCATION tango_cache_object_locate(struct tango_cache_instance *instance, size_t object_size);
void tango_cache_get_object_path(struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize);
struct tango_cache_ctx *tango_cache_update_once_prepare(struct tango_cache_instance *instance,
struct future* f, struct tango_cache_meta_put *meta, size_t object_size, char *path, size_t pathsize);
#endif