gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-18286 Proxy支持虚拟表表名变更，删除代理本地缓存，删除tsg-http相关配置

Browse Source
This commit is contained in:
fengweihao
2023-12-29 14:39:03 +08:00
parent b011a92680
commit ced991b4a0
54 changed files with 34 additions and 8443 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@@ -97,6 +97,5 @@ add_subdirectory(vendor)
add_subdirectory(common)
add_subdirectory(plugin)
add_subdirectory(platform)
add_subdirectory(cache)
add_subdirectory(script)
add_subdirectory(bpf)

5
cache/CMakeLists.txt vendored
View File

@@ -1,5 +0,0 @@
add_library(tango-cache-client src/object_store_client.cpp src/cache_evbase_client.cpp src/tango_cache_client.cpp src/tango_cache_redis.cpp src/tango_cache_pending.cpp src/tango_cache_tools.cpp src/tango_cache_transfer.cpp src/tango_cache_xml.cpp)
target_link_libraries(tango-cache-client http)
target_include_directories(tango-cache-client PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
target_link_libraries(tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries(tango-cache-client MESA_handle_logger MESA_htable MESA_prof_load wiredLB)

12
cache/README.txt vendored
View File

@@ -1,12 +0,0 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD>Minio<EFBFBD>ͼ<EFBFBD>Ⱥ<EFBFBD><EFBFBD>Redis<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ļ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͻ<EFBFBD><EFBFBD>ˣ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>redis<EFBFBD><EFBFBD>ȡ<EFBFBD>Ǹÿͻ<EFBFBD><EFBFBD><EFBFBD>ʵ<EFBFBD>֣<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Minio<EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD>֪ͨredis<EFBFBD><EFBFBD>
1<EFBFBD><EFBFBD>֧<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>»<EFBFBD><EFBFBD>淽ʽ
<09><>1<EFBFBD><31>Ԫ<EFBFBD><D4AA>Ϣ<EFBFBD>Ͷ<EFBFBD><CDB6><EFBFBD>ȫ<EFBFBD><C8AB><EFBFBD>洢<EFBFBD><E6B4A2>Minio<69><6F>
<09><>2<EFBFBD><32>Ԫ<EFBFBD><D4AA>Ϣ<EFBFBD>洢<EFBFBD><E6B4A2>Redis<69><73><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>洢<EFBFBD><E6B4A2>Minio<69><6F>˳<EFBFBD><CBB3>Ҳ<EFBFBD>洢<EFBFBD><E6B4A2>Ԫ<EFBFBD><D4AA>Ϣ<EFBFBD><CFA2><EFBFBD><EFBFBD>HEAD<41><44><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Redisȡ<73><C8A1><EFBFBD><EFBFBD>
<09><>3<EFBFBD><33>Ԫ<EFBFBD><D4AA>Ϣ<EFBFBD><CFA2>С<EFBFBD>ļ<EFBFBD><C4BC>洢<EFBFBD><E6B4A2>Redis<69><73><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC>洢<EFBFBD><E6B4A2>Minio<69><6F><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC>趨<EFBFBD><E8B6A8><EFBFBD><EFBFBD>
2<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>İ汾<EFBFBD><EFBFBD><EFBFBD>£<EFBFBD>
libevent<6E><74>2.1<EFBFBD>Ժ<EFBFBD><EFBFBD>汾<EFBFBD><EFBFBD>
libcurl<72><6C> 7.43<EFBFBD>Ժ<EFBFBD><EFBFBD>汾<EFBFBD><EFBFBD><EFBFBD>Ƽ<EFBFBD><EFBFBD>汾7.59<EFBFBD><EFBFBD>һ<EFBFBD><EFBFBD>Ҫע<EFBFBD><EFBFBD><EFBFBD>汾<EFBFBD><EFBFBD><EFBFBD>⣡<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
libxml2: 2.6<EFBFBD>Ժ<EFBFBD><EFBFBD>汾
hiredis-vip<69><70>0.3.0<EFBFBD><EFBFBD><EFBFBD>ÿ<EFBFBD><EFBFBD>Ǿ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>޸ĵİ汾<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>صIJ<EFBFBD><EFBFBD><EFBFBD>ֱ<EFBFBD><EFBFBD>ʹ<EFBFBD>ã<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
3<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ο<EFBFBD>ͷ<EFBFBD>ļ<EFBFBD>tango_cache_client.h<><68><EFBFBD>ǽṹ<C7BD><E1B9B9><EFBFBD><EFBFBD>־API<50>ο<EFBFBD>ͷ<EFBFBD>ļ<EFBFBD>cache_evbase_client.h<><68>

67
cache/include/cache_evbase_client.h vendored
View File

@@ -1,67 +0,0 @@
#ifndef __CACHE_ASYN_CLIENT_H__
#define __CACHE_ASYN_CLIENT_H__
#include <event2/event.h>
#include <event.h>
#include "tango_cache_client.h"
/* API<50><49>ʹ<EFBFBD><CAB9>˵<EFBFBD><CBB5><EFBFBD>ο<EFBFBD>tango_cache_client.h */
struct cache_evbase_instance
{
struct tango_cache_instance *instance;
evutil_socket_t notify_readfd;
evutil_socket_t notify_sendfd;
struct event_base* evbase;
};
struct cache_evbase_ctx
{
size_t object_size; //tango_ctx<74><78><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD>޷<EFBFBD><DEB7><EFBFBD><EFBFBD>ع<EFBFBD><D8B9><EFBFBD><EFBFBD><EFBFBD>ֱ<EFBFBD>ӻ<EFBFBD>ȡ<EFBFBD>Ļ<EFBFBD><C4BB><EFBFBD><EFBFBD>ڶ<EFBFBD>дһ<D0B4><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
struct tango_cache_ctx *ctx;
struct cache_evbase_instance *instance_asyn;
};
/*<2A><><EFBFBD><EFBFBD>API<50>̰߳<DFB3>ȫ<EFBFBD><C8AB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬһ<CDAC><D2BB>cache_evbase_ctx<74><78><EFBFBD><EFBFBD><EFBFBD>Կ<EFBFBD><D4BF>̷߳<DFB3><CCB7>ʡ<EFBFBD>*/
enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn);
enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance);
void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out);
void cache_evbase_global_init(void);
//ÿ<><C3BF>minio<69><6F>Ⱥ<EFBFBD><C8BA>bucket<65><74><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>parameter<65><72><EFBFBD><EFBFBD><EFBFBD><EFBFBD>instance<63>ɹ<EFBFBD><C9B9><EFBFBD>һ<EFBFBD><D2BB>parameter<65><72>
struct tango_cache_parameter *cache_evbase_parameter_new(const char* profile_path, const char* section, void *runtimelog);
/*<2A><><EFBFBD><EFBFBD>ʵ<EFBFBD><CAB5><EFBFBD><EFBFBD><EFBFBD>̰߳<DFB3>ȫ<EFBFBD><C8AB><EFBFBD>ڲ<EFBFBD><DAB2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>߳<EFBFBD>*/
struct cache_evbase_instance *cache_evbase_instance_new(struct tango_cache_parameter *param, void *runtimelog);
/******************************************* GET<45>ӿ<EFBFBD> ****************************************/
//<2F>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD>0<EFBFBD><30>ʧ<EFBFBD>ܷ<EFBFBD><DCB7><EFBFBD>-1<><31>future<72>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>߳<EFBFBD><DFB3><EFBFBD>ִ<EFBFBD>У<EFBFBD><D0A3><EFBFBD>ͬ
int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get);
int cache_evbase_head_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta);
struct tango_cache_result *cache_evbase_read_result(void *promise_result);
/****************************************** DELETE<54>ӿ<EFBFBD> **************************************/
int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* f, const char *objkey, const char *minio_addr=NULL, const char *bucket=NULL);
/***************************************** һ<><D2BB><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD> **********************************/
int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* f,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size,
struct tango_cache_meta_put *meta,
char *path/*OUT*/, size_t pathsize);
int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struct future* f,
struct evbuffer *evbuf,
struct tango_cache_meta_put *meta,
char *path/*OUT*/, size_t pathsize);
/****************************************** <20><>ʽ<EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD> ***********************************/
struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_put *meta);
int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size);
int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf);
int cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn, char *path/*OUT*/, size_t pathsize);
void cache_evbase_update_cancel(struct cache_evbase_ctx *ctx_asyn);
#endif

52
cache/include/object_store_client.h vendored
View File

@@ -1,52 +0,0 @@
#ifndef __OBJECT_STORE_CLIENT_H__
#define __OBJECT_STORE_CLIENT_H__
#include <event2/event.h>
#include <event.h>
#include "cache_evbase_client.h"
struct object_store_instance
{
struct cache_evbase_instance **instances;
u_int32_t instance_num;
};
/*<2A><><EFBFBD><EFBFBD>API<50>̰߳<DFB3>ȫ<EFBFBD><C8AB>API<50><49>ʹ<EFBFBD><CAB9>˵<EFBFBD><CBB5><EFBFBD>ο<EFBFBD>tango_cache_client.h*/
enum CACHE_ERR_CODE object_store_get_last_error(const struct cache_evbase_ctx *ctx_asyn);
void object_store_get_statistics(const struct object_store_instance *instance, struct cache_statistics *out);
void object_store_global_init(void);
/*<2A><><EFBFBD><EFBFBD>ʵ<EFBFBD><CAB5><EFBFBD><EFBFBD><EFBFBD>̰߳<DFB3>ȫ<EFBFBD><C8AB><EFBFBD>ڲ<EFBFBD><DAB2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>߳<EFBFBD>*/
struct object_store_instance *object_store_instance_new(const char* profile_path, const char* section, int thread_num, void *runtimelog);
//GET<45>ӿڣ<D3BF><DAA3>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD>0<EFBFBD><30>ʧ<EFBFBD>ܷ<EFBFBD><DCB7><EFBFBD>-1<><31>future<72>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>߳<EFBFBD><DFB3><EFBFBD>ִ<EFBFBD>У<EFBFBD><D0A3><EFBFBD>ͬ
int object_store_fetch_object(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get);
int object_store_head_object(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_get *meta);
struct tango_cache_result *object_store_read_result(void *promise_result);
//DELETE<54>ӿ<EFBFBD>
int object_store_delete_object(struct object_store_instance *instance, struct future* f, const char *objkey, const char *minio_addr=NULL, const char *bucket=NULL);
//һ<><D2BB><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD>
int object_store_upload_once_data(struct object_store_instance *instance, struct future* f,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size,
struct tango_cache_meta_put *meta,
char *path/*OUT*/, size_t pathsize);
int object_store_upload_once_evbuf(struct object_store_instance *instance, struct future* f,
struct evbuffer *evbuf,
struct tango_cache_meta_put *meta,
char *path/*OUT*/, size_t pathsize);
//<2F><>ʽ<EFBFBD>ϴ<EFBFBD><CFB4>ӿ<EFBFBD>
struct cache_evbase_ctx *object_store_update_start(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_put *meta);
int object_store_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size);
int object_store_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf);
int object_store_update_end(struct cache_evbase_ctx *ctx_asyn, char *path/*OUT*/, size_t pathsize);
void object_store_update_cancel(struct cache_evbase_ctx *ctx_asyn);
#endif

175
cache/include/tango_cache_client.h vendored
View File

@@ -1,175 +0,0 @@
#ifndef __TANGO_CACHE_CLIENT_H__
#define __TANGO_CACHE_CLIENT_H__
#include <event2/event.h>
#include <event.h>
#include <tfe_future.h>
#include "tango_cache_pending.h"
#define USER_TAG_MAX_LEN 1518
enum CACHE_ERR_CODE
{
CACHE_OK=0,
CACHE_CACHE_MISS = -101, //<2F><><EFBFBD><EFBFBD>δ<EFBFBD><CEB4><EFBFBD><EFBFBD>
CACHE_TIMEOUT = -102, //<2F><><EFBFBD>泬ʱ
CACHE_OUTOF_MEMORY= -103,//<2F><>ǰ<EFBFBD>ڴ<EFBFBD>ռ<EFBFBD>ó<EFBFBD><C3B3><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƣ<EFBFBD><C6A3>鿴MAX_USED_MEMORY_SIZE_MB<4D>Ƿ<EFBFBD><C7B7><EFBFBD>С<EFBFBD><D0A1><EFBFBD>ߵ<EFBFBD>ǰ<EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD>ʸ<EFBFBD><CAB8><EFBFBD><EFBFBD>ϵ<EFBFBD><CFB5><EFBFBD><EFBFBD>ߵ<EFBFBD><DFB5><EFBFBD><EFBFBD><EFBFBD>
CACHE_ERR_CURL = -104,
CACHE_ERR_WIREDLB = -105,
CACHE_ERR_SOCKPAIR= -106,
CACHE_ERR_INTERNAL= -107,
CACHE_ERR_REDIS_JSON = -108,
CACHE_ERR_REDIS_CONNECT= -109,
CACHE_ERR_REDIS_EXEC = -110,
CACHE_OUTOF_SESSION = -111,
CACHE_ERR_EVBUFFER = -112,
CACHE_UPDATE_CANCELED = -113,
};
struct cache_statistics
{
long long get_recv_num; //<2F><><EFBFBD><EFBFBD>GET<45>Ĵ<EFBFBD><C4B4><EFBFBD>
long long get_succ_http; //GET minio<69>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long get_succ_redis;//GET redis<69>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long get_miss_num; //GETδ<54><CEB4><EFBFBD>еĴ<D0B5><C4B4><EFBFBD>
long long get_err_http; //GET minioʧ<6F>ܵĴ<DCB5><C4B4><EFBFBD>
long long get_err_redis; //GET redisʧ<73>ܵĴ<DCB5><C4B4><EFBFBD>
long long put_recv_num; //<2F><><EFBFBD><EFBFBD>UPLOAD<41>Ĵ<EFBFBD><C4B4><EFBFBD>
long long put_succ_http; //UPLOAD minio<69>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long put_succ_redis;//UPLOAD redis<69>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long put_err_http; //UPLOAD minioʧ<6F>ܵĴ<DCB5><C4B4><EFBFBD>
long long put_err_redis; //UPLOAD redisʧ<73>ܵĴ<DCB5><C4B4><EFBFBD>
long long del_recv_num; //<2F><><EFBFBD><EFBFBD>DELETE<54>Ĵ<EFBFBD><C4B4><EFBFBD>
long long del_succ_num; //DELETE<54>ɹ<EFBFBD><C9B9>Ĵ<EFBFBD><C4B4><EFBFBD>
long long del_error_num; //DELETEʧ<45>ܵĴ<DCB5><C4B4><EFBFBD>
long long totaldrop_num; //<2F>ڴ<EFBFBD><DAB4><EFBFBD><EFBFBD>Լ<EFBFBD>WiredLB<4C><42><EFBFBD><EFBFBD>ʱDROP<4F>Ĵ<EFBFBD><C4B4><EFBFBD>
long long memory_used; //<2F><>ǰUPLOAD BODY<44><59>ռ<EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD>С
long long session_http; //<2F><>ǰ<EFBFBD><C7B0><EFBFBD>ڽ<EFBFBD><DABD><EFBFBD>GET/PUT<55><54>HTTP<54>Ự<EFBFBD><E1BBB0>
long long session_redis; //<2F><>ǰ<EFBFBD><C7B0><EFBFBD>ڽ<EFBFBD><DABD><EFBFBD>GET/PUT<55><54>redis<69>Ự<EFBFBD><E1BBB0>
};
struct tango_cache_parameter;
struct tango_cache_instance;
struct tango_cache_ctx;
enum CACHE_ERR_CODE tango_cache_get_last_error(const struct tango_cache_ctx *ctx);
enum CACHE_ERR_CODE tango_cache_ctx_error(const struct tango_cache_instance *instance);
void tango_cache_get_statistics(const struct tango_cache_instance *instance, struct cache_statistics *out);
/*ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ִ<EFBFBD><D6B4>һ<EFBFBD>γ<EFBFBD>ʼ<EFBFBD><CABC>*/
void tango_cache_global_init(void);
//ÿ<><C3BF>minio<69><6F>Ⱥ<EFBFBD><C8BA>bucket<65><74><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>parameter<65><72><EFBFBD><EFBFBD><EFBFBD><EFBFBD>instance<63>ɹ<EFBFBD><C9B9><EFBFBD>һ<EFBFBD><D2BB>parameter
struct tango_cache_parameter *tango_cache_parameter_new(const char* profile_path, const char* section, void *runtimelog);
/*<2A><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>API<50>̲߳<DFB3><CCB2><EFBFBD>ȫ*/
//ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>̴߳<DFB3><CCB4><EFBFBD>һ<EFBFBD><D2BB>instance
struct tango_cache_instance *tango_cache_instance_new(struct tango_cache_parameter *param, struct event_base* evbase, void *runtimelog);
/****************************************** GET<45>ӿڵ<D3BF>API ******************************************/
enum CACHE_RESULT_TYPE
{
RESULT_TYPE_HEADER=0, //<2F><><EFBFBD><EFBFBD>ֻ<EFBFBD><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>
RESULT_TYPE_USERTAG, //<2F><><EFBFBD><EFBFBD>ֻ<EFBFBD><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>
RESULT_TYPE_BODY, //<2F><><EFBFBD>ܵ<EFBFBD><DCB5>ö<EFBFBD><C3B6><EFBFBD>
RESULT_TYPE_END, //ȫ<><C8AB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֻ<EFBFBD><D6BB><EFBFBD><EFBFBD>һ<EFBFBD>Σ<EFBFBD><CEA3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
RESULT_TYPE_MISS, //<2F><><EFBFBD><EFBFBD>δ<EFBFBD><CEB4><EFBFBD>У<EFBFBD><D0A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͻ<EFBFBD><CDBB>⣬ֻ<E2A3AC><D6BB><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>(<28><>END֮<44><D6AE>)<29><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
};
enum OBJECT_LOCATION
{
OBJECT_IN_UNKNOWN=0,
OBJECT_IN_HOS,
OBJECT_IN_REDIS
};
struct tango_cache_meta_get
{
const char* url; //<2F><><EFBFBD><EFBFBD>:URL<52><4C><EFBFBD>ǽṹ<C7BD><E1B9B9><EFBFBD><EFBFBD>־:<3A>ļ<EFBFBD>·<EFBFBD><C2B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>CACHE_OBJECT_KEY_HASH_SWITCH=0ʱ<30><CAB1><EFBFBD>󳤶<EFBFBD>256<35>ֽڣ<D6BD>=1ʱ<31><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
struct request_freshness get;
};
//promise_success<73>Ľ<EFBFBD><C4BD><EFBFBD>result
struct tango_cache_result
{
const char *data_frag; //<2F><><EFBFBD><EFBFBD>typeΪRESULT_TYPE_HEADER<45><52>ÿ<EFBFBD><C3BF>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>(HTTP1.1<EFBFBD><EFBFBD>ʽ)
size_t size;
size_t tlength; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ܳ<EFBFBD><DCB3>ȣ<EFBFBD><C8A3>ص<EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>Ч
enum CACHE_RESULT_TYPE type;
enum OBJECT_LOCATION location;
};
//<2F>ɹ<EFBFBD>ʱ<EFBFBD>ص<EFBFBD>promise_success
//ʧ<><CAA7>ʱ<EFBFBD>ص<EFBFBD>promise_failed(<28><>һ<EFBFBD><D2BB>)<29><>ʹ<EFBFBD><CAB9>get_last_error<6F><72>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD>룻
//future<72><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ΪNULL
int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get);
int tango_cache_head_object(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_get *meta);
//<2F><>promise_success<73><73>result<6C><74><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD>
struct tango_cache_result *tango_cache_read_result(future_result_t *promise_result);
/****************************************** DELETE<54>ӿڵ<D3BF>API ******************************************/
int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* f, const char *objkey, const char *minio_addr=NULL, const char *bucket=NULL);
/****************************************** UPLOAD<41>ӿڵ<D3BF>API ******************************************/
/* ע<><D7A2>: <20><>future<72><65>ΪNULL<4C><4C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>֪ͨ<CDA8>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>򲻵<EFBFBD><F2B2BBB5>ã<EFBFBD>*/
enum PUT_MEMORY_COPY_WAY
{
PUT_MEM_COPY=0, //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڴ<EFBFBD>
PUT_MEM_FREE, //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڴ棬<DAB4><E6A3AC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɱ<EFBFBD><C9B1><EFBFBD><EFBFBD><EFBFBD>ģ<EFBFBD><C4A3><EFBFBD>ͷŸ<CDB7><C5B8>ڴ<EFBFBD>
};
enum EVBUFFER_COPY_WAY
{
EVBUFFER_MOVE=0,//evbuffer_add_buffer
EVBUFFER_COPY, //evbuffer_add_buffer_reference
};
enum CACHE_HTTP_HDR_TYPE
{
HDR_CONTENT_TYPE=0,
HDR_CONTENT_ENCODING,
HDR_CONTENT_DISPOSITION,
HDR_CONTENT_MD5,
HDR_CONTENT_NUM,
};
struct tango_cache_meta_put
{
const char* url;
const char* std_hdr[HDR_CONTENT_NUM]; //<2F><><EFBFBD><EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"Content-Type: text/html"<22><><EFBFBD><EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>У<EFBFBD>NULL<4C><4C>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7><EFBFBD><EFBFBD>
const char* usertag; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD>GETʱ<54><CAB1>ԭ<EFBFBD><D4AD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
size_t usertag_len; //<2F><><EFBFBD>󳤶<EFBFBD>USER_TAG_MAX_LEN<45><4E>0<EFBFBD><30>ʾû<CABE>и<EFBFBD>ͷ<EFBFBD><CDB7>
size_t user_log_name;
struct response_freshness put;
};
/****************************************** <20><><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>UPLOAD<41>ӿڵ<D3BF>API ******************************************/
//<2F><>path<74><68>Ϊ<EFBFBD>գ<EFBFBD><D5A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĵ洢·<E6B4A2><C2B7>
//<2F><><EFBFBD><EFBFBD>ֵ: 0-<2D>ɹ<EFBFBD><C9B9><EFBFBD><0ʧ<30>ܣ<EFBFBD><DCA3><EFBFBD>ͬ
int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct future* f,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size,
struct tango_cache_meta_put *meta,
char *path/*OUT*/, size_t pathsize);
int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct future* f,
enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf,
struct tango_cache_meta_put *meta,
char *path/*OUT*/, size_t pathsize);
/****************************************** <20><>ʽUPLOAD<41>ӿڵ<D3BF>API ******************************************/
//<2F><><EFBFBD><EFBFBD>ֵ: <20><>ΪNULL<4C><4C><EFBFBD><EFBFBD>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD>ʧ<EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD>tango_cache_ctx_error<6F>鿴<EFBFBD><E9BFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>CACHE_OUTOF_MEMORY(<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>)<29><>
struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_put *meta);
//<2F><><EFBFBD><EFBFBD>ֵ: 0-<2D>ɹ<EFBFBD><C9B9><EFBFBD><0ʧ<30>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD>tango_cache_get_last_error<6F>鿴<EFBFBD><E9BFB4><EFBFBD><EFBFBD><EFBFBD>룻
int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data, size_t size);
int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf);
//ע<><D7A2>: <20><><EFBFBD><EFBFBD>ʧ<EFBFBD><CAA7>ʱ<EFBFBD><CAB1><EFBFBD>ٵ<EFBFBD><D9B5><EFBFBD>promise<73>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>path<74><68>ΪNULLʱ<4C><CAB1><EFBFBD>ش洢·<E6B4A2><C2B7>
int tango_cache_update_end(struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize);
//<2F><><EFBFBD><EFBFBD>cancel<65>󲻻<EFBFBD><F3B2BBBB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʧ<EFBFBD><CAA7>
void tango_cache_update_cancel(struct tango_cache_ctx *ctx);
#endif

64
cache/include/tango_cache_pending.h vendored
View File

@@ -1,64 +0,0 @@
#pragma once
#include<time.h>
#include<tfe_http.h>
enum cache_pending_action {
UNDEFINED = 0,
ALLOWED,
FORBIDDEN,
REVALIDATE
};
struct tfe_http_field {
enum tfe_http_std_field http_field;
const char* value;
};
struct request_freshness {
time_t min_fresh;
time_t max_age;
};
struct response_freshness{
time_t date;
time_t last_modified;
time_t timeout;
};
time_t read_GMT_time(const char* gmt_string);
/*
函数功能：
根据请求头字段判断是否允许将缓存作为该请求的响应，并且将请求字段对缓存新鲜度的约束范围作为传出参数返回给调用者
参数:
request:HTTP请求字段信息，包括Pragma,Cache-Control,If-Match,If-None-Match,If-Modified-Since,If-Unmodified-Since字段
n_fields:request包含的HTTP请求字段数目
restrict:如果该函数返回值为ALLOWED，则返回请求Cache-Control字段包括的min-fresh或者max-age值；否则返回值为0
返回值：
UNDEFINED = 0,//请求字段中未定义缓存的行为
ALLOWED ,//允许使用缓存作为该请求的响应
FORBIDDEN,//禁止使用缓存作为该请求的响应，需要向源服务器请求
REVALIDATE,//禁止使用未验证有效性的缓存作为该请求的响应
*/
enum cache_pending_action tfe_cache_get_pending(const struct tfe_http_half *request, struct request_freshness* restrict);
/*
函数功能：
根据响应字段判断该响应是否允许被缓存，并且将响应的新鲜度、date或者last-modified值作为传出参数返回给调用者
参数:
response:HTTP响应字段信息，包括Pragma,Cache-Control,Expires、Date、Last-Modified字段
n_fields:response包含的HTTP响应字段数目
freshness:如果该函数返回值为ALLOWED，则返回响应Cache-Control字段的s-maxage/max-age值以及Date字段、Last-Modified字段相对时间
返回值：
UNDEFINED = 0,//响应字段中未定义缓存的行为
ALLOWED ,//允许缓存该响应
FORBIDDEN,//禁止缓存该响应
*/
enum cache_pending_action tfe_cache_put_pending(const struct tfe_http_half *response, struct response_freshness* freshness);

640
cache/src/cache_evbase_client.cpp vendored
View File

@@ -1,640 +0,0 @@
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/prctl.h>
#include <string.h>
#include <pthread.h>
#include "cache_evbase_client.h"
#include "tango_cache_transfer.h"
#include "tango_cache_tools.h"
enum CACHE_ASYN_CMD
{
CACHE_ASYN_FETCH=0,
CACHE_ASYN_UPLOAD_ONCE_DATA,
CACHE_ASYN_UPLOAD_ONCE_EVBUF,
CACHE_ASYN_UPLOAD_START,
CACHE_ASYN_UPLOAD_FRAG_DATA,
CACHE_ASYN_UPLOAD_FRAG_EVBUF,
CACHE_ASYN_UPLOAD_END,
CACHE_ASYN_UPLOAD_CANCEL,
CACHE_ASYN_DELETE,
CACHE_ASYN_HEAD,
};
struct databuffer
{
char *data;
size_t size;
struct evbuffer *evbuf;
enum CACHE_ASYN_CMD cmd_type;
enum OBJECT_LOCATION where_to_get;
struct cache_evbase_ctx *ctx_asyn;
};
enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn)
{
return tango_cache_get_last_error(ctx_asyn->ctx);
}
enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance)
{
return tango_cache_ctx_error(instance->instance);
}
void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out)
{
tango_cache_get_statistics(instance->instance, out);
}
struct tango_cache_result *cache_evbase_read_result(void *promise_result)
{
return tango_cache_read_result(promise_result);
}
static int create_notification_pipe(evutil_socket_t sv[2])
{
if(evutil_socketpair(AF_LOCAL, SOCK_STREAM, 0, sv) == -1)
{
return -1;
}
if(evutil_make_socket_nonblocking(sv[0])<0 || evutil_make_socket_nonblocking(sv[1])<0)
{
return -1;
}
if(evutil_make_socket_closeonexec(sv[0])<0 || evutil_make_socket_closeonexec(sv[1])<0)
{
return -1;
}
return 0;
}
static int32_t mesa_tcp_sock_write (int32_t write_fd, void *buf, int32_t bufsize)
{
int32_t res;
do{
res = send(write_fd, buf, bufsize, MSG_NOSIGNAL);
}while (res==-1 &&(errno == EINTR));
return res;
}
static int32_t iothread_notify_event(int32_t socket_fd, void *content, int32_t len, int32_t s_time_out)
{
fd_set w_set, e_set;
struct timeval tv;
int32_t res=0, sndlen=0, sendsize=0;
while(len > sndlen)
{
FD_ZERO (&w_set);
FD_ZERO (&e_set);
FD_SET (socket_fd, &w_set);
FD_SET (socket_fd, &e_set);
if(s_time_out == 0)
{
res = select (socket_fd + 1, NULL, &w_set, &e_set, NULL);
}
else
{
tv.tv_sec = s_time_out;
tv.tv_usec = 0;
res = select (socket_fd + 1, NULL, &w_set, &e_set, &tv);
}
if(res <= 0)
{
printf("log_error: select io res=%d, error: %s\n", res, strerror(errno));
return -1;
}
if(FD_ISSET(socket_fd, &e_set))
{
printf("log_error: select io is in efds, error: %s\n", strerror(errno));
return -2;
}
if(FD_ISSET(socket_fd, &w_set))
{
sendsize = mesa_tcp_sock_write(socket_fd, (char*)content + sndlen, len - sndlen);
if (sendsize < 0)
{
if(errno == EAGAIN)
{
continue;
}
return -1;
}
sndlen += sendsize;
}
}
return sndlen;
}
static void cache_asyn_ctx_destroy(struct cache_evbase_ctx *ctx_asyn)
{
free(ctx_asyn);
}
static void cache_asyn_ioevent_dispatch(struct databuffer *buffer)
{
struct cache_evbase_ctx *ctx_asyn=buffer->ctx_asyn;
struct promise *p;
int ret=0;
switch(buffer->cmd_type)
{
case CACHE_ASYN_FETCH:
p = ctx_asyn->ctx->promise;
if(do_tango_cache_fetch_object(ctx_asyn->ctx, buffer->where_to_get) < 0)
{
promise_failed(p, FUTURE_ERROR_CANCEL, "CACHE_ASYN_FETCH failed");
}
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_HEAD:
p = ctx_asyn->ctx->promise;
ret = do_tango_cache_head_object(ctx_asyn->ctx, buffer->where_to_get);
if(ret<0)
{
promise_failed(p, FUTURE_ERROR_CANCEL, "CACHE_ASYN_HEAD failed");
}
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_DELETE:
cache_delete_minio_object(ctx_asyn->ctx, true);
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_UPLOAD_ONCE_DATA:
do_tango_cache_upload_once_data(ctx_asyn->ctx, PUT_MEM_FREE, buffer->data, buffer->size, true);
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_UPLOAD_ONCE_EVBUF:
do_tango_cache_upload_once_evbuf(ctx_asyn->ctx, EVBUFFER_MOVE, buffer->evbuf, true);
evbuffer_free(buffer->evbuf);
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_UPLOAD_START:
ctx_asyn->ctx->instance->statistic.put_recv_num += 1;
ctx_asyn->ctx->instance->error_code = CACHE_OK;
break;
case CACHE_ASYN_UPLOAD_FRAG_DATA:
tango_cache_update_frag_data(ctx_asyn->ctx, buffer->data, buffer->size);
free(buffer->data);
break;
case CACHE_ASYN_UPLOAD_FRAG_EVBUF:
tango_cache_update_frag_evbuf(ctx_asyn->ctx, EVBUFFER_MOVE, buffer->evbuf);
evbuffer_free(buffer->evbuf);
break;
case CACHE_ASYN_UPLOAD_END:
do_tango_cache_update_end(ctx_asyn->ctx, true);
cache_asyn_ctx_destroy(ctx_asyn);
break;
case CACHE_ASYN_UPLOAD_CANCEL:
tango_cache_update_cancel(ctx_asyn->ctx);
cache_asyn_ctx_destroy(ctx_asyn);
break;
default: assert(0);break;
}
}
static void sockpair_notification_handler(evutil_socket_t fd, short events, void *arg)
{
ssize_t readlen, needlen;
struct cache_evbase_instance *instance_asyn = (struct cache_evbase_instance *)arg;
struct databuffer *buffer;
while(1)
{
needlen=sizeof(struct cache_evbase_ctx *);
readlen = recv(fd, &buffer, needlen, 0);
if(readlen == needlen)
{
cache_asyn_ioevent_dispatch(buffer);
free(buffer);
}
else
{
if(errno!=EWOULDBLOCK && errno!=EAGAIN)
{
MESA_HANDLE_RUNTIME_LOGV2(instance_asyn->instance->runtime_log, RLOG_LV_FATAL, "read pipe error: %s.", strerror(errno));
assert(0);
return;
}
break;
}
}
}
static void* thread_listen_sockpair(void *arg)
{
struct cache_evbase_instance *instance_asyn = (struct cache_evbase_instance *)arg;
struct event listen_event;
prctl(PR_SET_NAME, "tango_cache");
event_assign(&listen_event, instance_asyn->evbase, instance_asyn->notify_readfd, EV_READ|EV_PERSIST, sockpair_notification_handler, instance_asyn);
event_add(&listen_event, NULL);
event_base_dispatch(instance_asyn->evbase);
printf("Libevent dispath error, should not run here.\n");
MESA_HANDLE_RUNTIME_LOGV2(instance_asyn->instance->runtime_log, RLOG_LV_FATAL, "Libevent dispath error, should not run here.");
assert(0);
return NULL;
}
int cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn, char *path/*OUT*/, size_t pathsize)
{
struct databuffer *buffer;
if(ctx_asyn->ctx->fail_state)
{
tango_cache_ctx_destroy(ctx_asyn->ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
return -1;
}
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_END;
//ENDʱ<44><CAB1><EFBFBD><EFBFBD>δ<EFBFBD><CEB4>ʼ<EFBFBD>ֶ<EFBFBD><D6B6>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD>֮ǰ<D6AE><C7B0><EFBFBD><EFBFBD>locateһ<65><D2BB>λ<EFBFBD><CEBB>
ctx_asyn->ctx->locate = tango_cache_object_locate(ctx_asyn->ctx->instance, ctx_asyn->object_size);
tango_cache_get_object_path(ctx_asyn->ctx, path, pathsize);
if(ctx_asyn->ctx->instance->param->object_store_way != CACHE_ALL_HOS)
{
cJSON_AddNumberToObject(ctx_asyn->ctx->put.object_meta, "Content-Length", ctx_asyn->object_size);
}
if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR);
tango_cache_ctx_destroy(ctx_asyn->ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
free(buffer);
return -2;
}
return 0;
}
void cache_evbase_update_cancel(struct cache_evbase_ctx *ctx_asyn)
{
struct databuffer *buffer;
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_CANCEL;
if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
if(!ctx_asyn->ctx->fail_state)
{
tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR);
}
tango_cache_ctx_destroy(ctx_asyn->ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
free(buffer);
}
}
int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size)
{
struct databuffer *buffer;
if(ctx_asyn->ctx->fail_state)
{
if(way == PUT_MEM_FREE) free((void *)data);
return 0;//<2F><>ʱ<EFBFBD>Ⱥ<EFBFBD><C8BA>Է<EFBFBD><D4B7><EFBFBD>ֵ<EFBFBD><D6B5><EFBFBD>Իص<D4BB><D8B5><EFBFBD>ʽ֪ͨ<CDA8><D6AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>û<EFBFBD><C3BB><EFBFBD>֪<EFBFBD><D6AA>ʱEND<4E><44><EFBFBD><EFBFBD><EFBFBD>⡣
}
ctx_asyn->object_size += size;
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
if(way == PUT_MEM_COPY)
{
buffer->data = (char *)malloc(size);
memcpy(buffer->data, data, size);
}
else
{
buffer->data = (char*)data;
}
buffer->size = size;
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_FRAG_DATA;
if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR);
free(buffer->data);
free(buffer);
return -2;
}
return 0;
}
int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf)
{
struct databuffer *buffer;
if(ctx_asyn->ctx->fail_state)
{
return 0;
}
ctx_asyn->object_size += evbuffer_get_length(evbuf);
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_FRAG_EVBUF;
buffer->evbuf = evbuffer_new();
evbuffer_add_buffer(buffer->evbuf, evbuf);
if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR);
evbuffer_free(buffer->evbuf);
free(buffer);
return -2;
}
return 0;
}
struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_put *meta)
{
struct cache_evbase_ctx *ctx_asyn;
struct tango_cache_ctx *ctx;
struct databuffer *buffer;
enum OBJECT_LOCATION maybe_loc=OBJECT_IN_UNKNOWN;
if(instance->instance->param->object_store_way != CACHE_SMALL_REDIS)
{
maybe_loc = OBJECT_IN_HOS;
}
ctx = tango_cache_update_prepare(instance->instance, f, meta, maybe_loc);
if(ctx == NULL)
{
return NULL;
}
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = ctx;
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_START;
//<2F>¼<EFBFBD>֪ͨ<CDA8><D6AA>Ϊ<EFBFBD><CEAA><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͳ<EFBFBD><CDB3><EFBFBD><EFBFBD>Ϣ
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
instance->instance->error_code = CACHE_ERR_SOCKPAIR;
tango_cache_set_fail_state(ctx, CACHE_ERR_SOCKPAIR);
tango_cache_ctx_destroy(ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
free(buffer);
return NULL;
}
return ctx_asyn;
}
int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* f,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, struct tango_cache_meta_put *meta, char *path, size_t pathsize)
{
struct cache_evbase_ctx *ctx_asyn;
struct tango_cache_ctx *ctx;
struct databuffer *buffer;
ctx = tango_cache_update_once_prepare(instance->instance, f, meta, size, path, pathsize);
if(ctx == NULL)
{
if(way == PUT_MEM_FREE) free((void *)data);
return -1;
}
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = ctx;
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
if(way == PUT_MEM_COPY)
{
buffer->data = (char *)malloc(size);
memcpy(buffer->data, data, size);
}
else
{
buffer->data = (char*)data;
}
buffer->size = size;
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_ONCE_DATA;
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
free(buffer->data);
free(buffer);
instance->instance->error_code = CACHE_ERR_SOCKPAIR;
tango_cache_set_fail_state(ctx, CACHE_ERR_SOCKPAIR);
tango_cache_ctx_destroy(ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
return -2;
}
return 0;
}
int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struct future* f,
struct evbuffer *evbuf, struct tango_cache_meta_put *meta, char *path, size_t pathsize)
{
struct cache_evbase_ctx *ctx_asyn;
struct tango_cache_ctx *ctx;
struct databuffer *buffer;
ctx = tango_cache_update_once_prepare(instance->instance, f, meta, evbuffer_get_length(evbuf), path, pathsize);
if(ctx == NULL)
{
return -1;
}
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = ctx;
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_UPLOAD_ONCE_EVBUF;
buffer->evbuf = evbuffer_new();
evbuffer_add_buffer(buffer->evbuf, evbuf);
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
evbuffer_free(buffer->evbuf);
free(buffer);
instance->instance->error_code = CACHE_ERR_SOCKPAIR;
tango_cache_set_fail_state(ctx, CACHE_ERR_SOCKPAIR);
tango_cache_ctx_destroy(ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
return -2;
}
return 0;
}
int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get)
{
struct cache_evbase_ctx *ctx_asyn;
struct databuffer *buffer;
if(instance->instance->param->object_store_way != CACHE_SMALL_REDIS)
{
where_to_get = OBJECT_IN_HOS;
}
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = tango_cache_fetch_prepare(instance->instance, CACHE_REQUEST_GET, f, meta, where_to_get);
if(ctx_asyn->ctx == NULL)
{
free(ctx_asyn);
return -1;
}
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_FETCH;
buffer->where_to_get = where_to_get;
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
instance->instance->error_code = CACHE_ERR_SOCKPAIR;
tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR);
tango_cache_ctx_destroy(ctx_asyn->ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
free(buffer);
return -2;
}
return 0;
}
int cache_evbase_head_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta)
{
struct cache_evbase_ctx *ctx_asyn;
struct databuffer *buffer;
enum OBJECT_LOCATION location = OBJECT_IN_HOS;
if(instance->instance->param->object_store_way != CACHE_ALL_HOS)
{
location = OBJECT_IN_REDIS;
}
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = tango_cache_fetch_prepare(instance->instance, CACHE_REQUEST_HEAD, f, meta, location);
if(ctx_asyn->ctx == NULL)
{
free(ctx_asyn);
return -1;
}
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_HEAD;
buffer->where_to_get = location;
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
instance->instance->error_code = CACHE_ERR_SOCKPAIR;
tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR);
tango_cache_ctx_destroy(ctx_asyn->ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
free(buffer);
return -2;
}
return 0;
}
int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket)
{
struct cache_evbase_ctx *ctx_asyn;
struct databuffer *buffer;
ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx));
ctx_asyn->instance_asyn = instance;
ctx_asyn->ctx = tango_cache_delete_prepare(instance->instance, f, objkey, minio_addr, bucket);
if(ctx_asyn->ctx == NULL)
{
free(ctx_asyn);
return -1;
}
buffer = (struct databuffer *)malloc(sizeof(struct databuffer));
buffer->ctx_asyn = ctx_asyn;
buffer->cmd_type = CACHE_ASYN_DELETE;
//<2F>ο<EFBFBD>Unix<69>߼<EFBFBD><DFBC><EFBFBD><EFBFBD><EFBFBD>432ҳ<32><D2B3><EFBFBD>ڶ<EFBFBD><DAB6>߳<EFBFBD>д<EFBFBD>İ<EFBFBD>ȫ<EFBFBD><C8AB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *))
{
instance->instance->error_code = CACHE_ERR_SOCKPAIR;
tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR);
tango_cache_ctx_destroy(ctx_asyn->ctx, false);
cache_asyn_ctx_destroy(ctx_asyn);
free(buffer);
return -2;
}
return 0;
}
struct tango_cache_parameter *cache_evbase_parameter_new(const char* profile_path, const char* section, void *runtimelog)
{
return tango_cache_parameter_new(profile_path, section, runtimelog);
}
struct cache_evbase_instance *cache_evbase_instance_new(struct tango_cache_parameter *param, void *runtimelog)
{
evutil_socket_t notification_fd[2];
struct cache_evbase_instance *instance_asyn;
struct event_base *evbase;
pthread_t thread_tid;
pthread_attr_t attr;
if(create_notification_pipe(notification_fd))
{
return NULL;
}
if((evbase = event_base_new()) == NULL)
{
return NULL;
}
instance_asyn = (struct cache_evbase_instance *)calloc(1, sizeof(struct cache_evbase_instance));
instance_asyn->evbase = evbase;
instance_asyn->notify_readfd = notification_fd[0];
instance_asyn->notify_sendfd = notification_fd[1];
instance_asyn->instance = tango_cache_instance_new(param, evbase, runtimelog);
if(instance_asyn->instance == NULL)
{
free(instance_asyn);
return NULL;
}
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
if(pthread_create(&thread_tid, &attr, thread_listen_sockpair, instance_asyn))
{
free(instance_asyn);
return NULL;
}
return instance_asyn;
}
void cache_evbase_global_init(void)
{
tango_cache_global_init();
}

145
cache/src/object_store_client.cpp vendored
View File

@@ -1,145 +0,0 @@
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/prctl.h>
#include <string.h>
#include <pthread.h>
#include "object_store_client.h"
#include "tango_cache_tools.h"
enum CACHE_ERR_CODE object_store_get_last_error(const struct cache_evbase_ctx *ctx)
{
return cache_evbase_get_last_error(ctx);
}
void object_store_get_statistics(const struct object_store_instance *instance, struct cache_statistics *out)
{
struct cache_statistics out_cache;
memset(out, 0, sizeof(struct cache_statistics));
for(u_int32_t i=0; i<instance->instance_num; i++)
{
cache_evbase_get_statistics(instance->instances[i], &out_cache);
out->del_error_num += out_cache.del_error_num;
out->del_recv_num += out_cache.del_recv_num;
out->del_succ_num += out_cache.del_succ_num;
out->get_err_http += out_cache.get_err_http;
out->get_err_redis += out_cache.get_err_redis;
out->get_miss_num += out_cache.get_miss_num;
out->get_recv_num += out_cache.get_recv_num;
out->get_succ_http += out_cache.get_succ_http;
out->get_succ_redis+= out_cache.get_succ_redis;
out->put_err_http += out_cache.put_err_http;
out->put_err_redis += out_cache.put_err_redis;
out->put_recv_num += out_cache.put_recv_num;
out->put_succ_http += out_cache.put_succ_http;
out->put_succ_redis+= out_cache.put_succ_redis;
out->session_http += out_cache.session_http;
out->session_redis += out_cache.session_redis;
out->memory_used += out_cache.memory_used;
out->totaldrop_num += out_cache.totaldrop_num;
}
}
struct tango_cache_result *object_store_read_result(void *promise_result)
{
return cache_evbase_read_result(promise_result);
}
int object_store_update_end(struct cache_evbase_ctx *ctx, char *path/*OUT*/, size_t pathsize)
{
return cache_evbase_update_end(ctx, path, pathsize);
}
void object_store_update_cancel(struct cache_evbase_ctx *ctx)
{
cache_evbase_update_cancel(ctx);
}
int object_store_update_frag_data(struct cache_evbase_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size)
{
return cache_evbase_update_frag_data(ctx, way, data, size);
}
int object_store_update_frag_evbuf(struct cache_evbase_ctx *ctx, struct evbuffer *evbuf)
{
return cache_evbase_update_frag_evbuf(ctx, evbuf);
}
struct cache_evbase_ctx *object_store_update_start(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_put *meta)
{
return cache_evbase_update_start(instance->instances[rand()%instance->instance_num], f, meta);
}
int object_store_upload_once_data(struct object_store_instance *instance, struct future* f,
enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, struct tango_cache_meta_put *meta, char *path, size_t pathsize)
{
return cache_evbase_upload_once_data(instance->instances[rand()%instance->instance_num], f, way, data, size, meta, path, pathsize);
}
int object_store_upload_once_evbuf(struct object_store_instance *instance, struct future* f,
struct evbuffer *evbuf, struct tango_cache_meta_put *meta, char *path, size_t pathsize)
{
return cache_evbase_upload_once_evbuf(instance->instances[rand()%instance->instance_num], f, evbuf, meta, path, pathsize);
}
int object_store_fetch_object(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get)
{
return cache_evbase_fetch_object(instance->instances[rand()%instance->instance_num], f, meta, where_to_get);
}
int object_store_head_object(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_get *meta)
{
return cache_evbase_head_object(instance->instances[rand()%instance->instance_num], f, meta);
}
int object_store_delete_object(struct object_store_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket)
{
return cache_evbase_delete_object(instance->instances[rand()%instance->instance_num], f, objkey, minio_addr, bucket);
}
struct object_store_instance *object_store_instance_new(const char* profile_path, const char* section, int thread_num, void *runtimelog)
{
struct object_store_instance *object_instance;
struct tango_cache_parameter *parameter;
parameter = tango_cache_parameter_new(profile_path, section, runtimelog);
if(parameter == NULL)
{
return NULL;
}
object_instance = (struct object_store_instance *)calloc(1, sizeof(struct object_store_instance));
object_instance->instance_num = thread_num;
object_instance->instances = (struct cache_evbase_instance **)calloc(1, sizeof(struct cache_evbase_instance *)*object_instance->instance_num);
for(u_int32_t i=0; i<object_instance->instance_num; i++)
{
object_instance->instances[i] = cache_evbase_instance_new(parameter, runtimelog);
if(object_instance->instances[i] == NULL)
{
free(parameter);
free(object_instance);
return NULL;
}
}
srandom(time(NULL));
return object_instance;
}
void object_store_global_init(void)
{
cache_evbase_global_init();
}

1251
cache/src/tango_cache_client.cpp vendored
View File

File diff suppressed because it is too large Load Diff

255
cache/src/tango_cache_client_in.h vendored
View File

@@ -1,255 +0,0 @@
#ifndef __TANGO_CACHE_CLIENT_IN_H__
#define __TANGO_CACHE_CLIENT_IN_H__
#include <curl/curl.h>
#include <sys/queue.h>
#include <pthread.h>
#include <event2/event.h>
#include <event.h>
#include <hiredis-vip/async.h>
#include <hiredis-vip/hircluster.h>
#include <cjson/cJSON.h>
#include <MESA/wiredLB.h>
#include <MESA/field_stat2.h>
#include "tango_cache_client.h"
#define RESPONSE_HDR_EXPIRES 1
#define RESPONSE_HDR_LAST_MOD 2
#define RESPONSE_HDR_ALL 3
#define CACHE_ALL_HOS 0 //Ԫ<><D4AA>Ϣ<EFBFBD>Ͷ<EFBFBD><CDB6>󶼴<EFBFBD><F3B6BCB4><EFBFBD>MINIO
#define CACHE_META_REDIS 1 //Ԫ<><D4AA>Ϣ<EFBFBD><CFA2>REDIS<49><53><EFBFBD><EFBFBD><EFBFBD><EFBFBD>MINIO
#define CACHE_SMALL_REDIS 2 //Ԫ<><D4AA>Ϣ<EFBFBD><CFA2>С<EFBFBD>ļ<EFBFBD><C4BC><EFBFBD>REDIS<49><53><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD>MINIO
enum FIELD_STAT_FILEDS
{
FS_FILED_GET_RECV=0,
FS_FILED_GET_S_TOTAL,
FS_FILED_GET_S_HTTP,
FS_FILED_GET_S_REDIS,
FS_FILED_GET_MISS,
FS_FILED_GET_E_TOTAL,
FS_FILED_GET_E_HTTP,
FS_FILED_GET_E_REDIS,
FS_FILED_PUT_RECV,
FS_FILED_PUT_S_TOTAL,
FS_FILED_PUT_S_HTTP,
FS_FILED_PUT_S_REDIS,
FS_FILED_PUT_E_TOTAL,
FS_FILED_PUT_E_HTTP,
FS_FILED_PUT_E_REDIS,
FS_FILED_DEL_RECV,
FS_FILED_DEL_SUCC,
FS_FILED_DEL_ERROR,
FS_FILED_TOTAL_DROP,
//Next use Status
FS_FILED_MEM_USED,
FS_FILED_SESS_HTTP,
FS_FILED_SESS_REDIS,
FS_FILED_NUM,
};
enum CACHE_REQUEST_METHOD
{
CACHE_REQUEST_GET=0,
CACHE_REQUEST_PUT,
CACHE_REQUEST_DELETE,
CACHE_REQUEST_DELETE_MUL,
CACHE_REQUEST_HEAD,
};
enum GET_OBJECT_STATE
{
GET_STATE_START=0,
GET_STATE_DELETE,
GET_STATE_REDIS_META,
GET_STATE_REDIS_ALL,
GET_STATE_REDIS_TRY,
GET_STATE_END,
};
enum PUT_OBJECT_STATE
{
PUT_STATE_START=0,
PUT_STATE_WAIT_START,
PUT_STATE_PART,
PUT_STATE_CANCEL,
PUT_STATE_REDIS_META,
PUT_STATE_REDIS_EXPIRE,
PUT_STATE_REDIS_SETEX, //<2F><>״̬<D7B4><CCAC><EFBFBD>ڵȴ<DAB5><C8B4><EFBFBD><EFBFBD><EFBFBD>ִ<EFBFBD>н<EFBFBD><D0BD><EFBFBD>
PUT_STATE_END,
};
struct easy_string
{
char* buff;
size_t len;
size_t size;
};
struct wiredlb_parameter
{
char wiredlb_topic[64];
char wiredlb_datacenter[64];
char wiredlb_group[64];
char iplist[4096];//minio<69>б<EFBFBD>
WLB_handle_t wiredlb;
u_int32_t wiredlb_override;
u_int32_t port;
short wiredlb_ha_port;
};
struct tango_cache_parameter
{
char bucketname[256];
char cache_token[256];
char redis_key[256];
long maximum_host_cnns;
long transfer_timeout;//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
long maximum_pipelines;
u_int64_t maximum_used_mem;
u_int32_t maximum_sessions;
u_int32_t upload_block_size; //minio<69>ֶ<EFBFBD><D6B6>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD><D0A1><EFBFBD><EFBFBD>
time_t relative_ttl; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ч<EFBFBD><D0A7>
u_int32_t hash_object_key;
//wiredlb
int object_store_way; //<2F><>ȡobject<63><74>Ϣ<EFBFBD>ķ<EFBFBD>ʽ
struct wiredlb_parameter cache;
char redisaddrs[4096];
u_int32_t redis_object_maxsize;//С<>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>redisʱ<73><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С
//FieldStatLog
int32_t fsstat_dst_port;
char fsstat_dst_ip[64];
char fsstat_appname[16];
char fsstat_filepath[256];
u_int32_t fsstat_period;
u_int32_t fsstatid_trig;
char fsstat_histlen[256];
screen_stat_handle_t fsstat_handle;
int32_t fsstat_histlen_id;
int32_t fsstat_field_ids[FS_FILED_NUM];
};
struct tango_cache_instance
{
struct event_base* evbase;
struct event timer_event;
struct event timer_statistic;
CURLM *multi_hd;
enum CACHE_ERR_CODE error_code;
int redis_connecting;
redisClusterAsyncContext *redis_ac;
char redisaddr[128];
const struct tango_cache_parameter *param;
void *runtime_log;
struct cache_statistics statistic;
struct cache_statistics statistic_last; //<2F><><EFBFBD>ڶ<EFBFBD><DAB6><EFBFBD>instanceʹ<65><CAB9>ͬһ<CDAC><D2BB>fieldstat<61>ۼ<EFBFBD>
};
struct multipart_etag_list
{
char *etag;
u_int32_t part_number;
TAILQ_ENTRY(multipart_etag_list) node;
};
typedef void (redisRedirectMinioCallback)(struct tango_cache_ctx *ctx);
struct cache_ctx_data_get
{
time_t max_age;
time_t min_fresh;
time_t expires;
time_t last_modify;
u_int32_t need_hdrs;
enum GET_OBJECT_STATE state;
struct easy_string response_tag;
struct tango_cache_result result;
redisRedirectMinioCallback *redis_redirect_minio_cb;
};
struct cache_ctx_data_put
{
struct evbuffer *evbuf;
size_t upload_length;
size_t upload_offset;
char *uploadID;
char *combine_xml;
TAILQ_HEAD(__etag_list_head, multipart_etag_list) etag_head;
cJSON *object_meta;
struct easy_string once_request; //һ<><D2BB><EFBFBD><EFBFBD>PUTʱ<54>洢<EFBFBD><E6B4A2><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD>ʧ<EFBFBD>ܵ<EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ܸ<EFBFBD><DCB8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ṹ
enum PUT_OBJECT_STATE state;
u_int32_t part_index; //<2F><>RESPONSE_HDR_
u_int32_t object_ttl;
bool close_state; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ùر<C3B9>
size_t object_size;
};
struct cache_ctx_multi_delete
{
u_int32_t succ_num;
u_int32_t fail_num;
};
struct tango_cache_ctx
{
CURL *curl;
struct curl_slist *headers;
struct promise* promise;
char error[CURL_ERROR_SIZE];
char object_key[256];
char hostaddr[48];
enum CACHE_REQUEST_METHOD method;
enum CACHE_ERR_CODE error_code;
struct easy_string response;
bool fail_state;
enum OBJECT_LOCATION locate; //<2F>ɳ<EFBFBD><C9B3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>϶<EFBFBD><CFB6><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD><CEBB>
long res_code;
union{
struct cache_ctx_data_put put;
struct cache_ctx_data_get get;
struct cache_ctx_multi_delete del;
};
struct tango_cache_instance *instance;
};
struct curl_socket_data
{
struct event sock_event;
};
void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_t size);
void easy_string_savedata(struct easy_string *estr, const char *data, size_t len);
void easy_string_destroy(struct easy_string *estr);
void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx, bool callback=true);
void tango_cache_set_fail_state(struct tango_cache_ctx *ctx, enum CACHE_ERR_CODE error_code);
const char *tango_cache_get_errstring(const struct tango_cache_ctx *ctx);
bool sessions_exceeds_limit(struct tango_cache_instance *instance, enum OBJECT_LOCATION where_to_get);
struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance,
struct future* f, struct tango_cache_meta_put *meta, enum OBJECT_LOCATION maybe_loc);
struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance,
enum CACHE_REQUEST_METHOD method, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get);
struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance,
struct future* f, const char *objkey, const char *minio_addr, const char *bucket);
enum OBJECT_LOCATION tango_cache_object_locate(struct tango_cache_instance *instance, size_t object_size);
void tango_cache_get_object_path(struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize);
struct tango_cache_ctx *tango_cache_update_once_prepare(struct tango_cache_instance *instance,
struct future* f, struct tango_cache_meta_put *meta, size_t object_size, char *path, size_t pathsize);
#endif

306
cache/src/tango_cache_pending.cpp vendored
View File

@@ -1,306 +0,0 @@
#include "tango_cache_pending.h"
#include <tfe_utils.h>
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
time_t get_time_value(const char* field_value, const char* field_type)
{
time_t time;
char* time_value = NULL;
field_value += strlen(field_type);
field_value++;
int len = strlen(field_value);
time_value = ALLOC(char, len+1);
int index = 0;
while (field_value[index] != ',' && field_value[index] != '\r' && index < len)
{
time_value[index] = field_value[index];
index++;
}
time_value[index] = '\0';
time = (time_t)atol(time_value);
free(time_value);
return time;
}
void get_request_freshness(const char *value, struct request_freshness* restrict)
{
const char* field_value = NULL;
field_value = strstr(value, "min-fresh");
if (field_value != NULL)
{
restrict->min_fresh = get_time_value(field_value, "min-fresh");;
}
field_value = strstr(value, "max-age");
if (field_value != NULL)
{
restrict->max_age = get_time_value(field_value, "max-age");;
}
}
enum cache_pending_action request_cache_control(const char* value, struct request_freshness* restrict)
{
if (strstr(value, "no-cache") != NULL)
{
return REVALIDATE;
}
if (strstr(value, "no-store") != NULL)
{
return FORBIDDEN;
}
get_request_freshness(value, restrict);
return ALLOWED;
}
bool cache_verify(const struct tfe_http_half *request)
{
if( !tfe_http_std_field_read(request,TFE_HTTP_IF_MATCH) ||
!tfe_http_std_field_read(request,TFE_HTTP_IF_NONE_MATCH) ||
!tfe_http_std_field_read(request,TFE_HTTP_IF_MODIFIED_SINCE) ||
!tfe_http_std_field_read(request,TFE_HTTP_IF_UNMODIFIED_SINCE)
)
{
return true;
}
return false;
}
const char* get_head_value(const struct tfe_http_field *http_fields, size_t n_fields, enum tfe_http_std_field head_key)
{
size_t i = 0;
for (i = 0; i < n_fields; i++)
{
if (http_fields[i].http_field == head_key)
{
return http_fields[i].value;
}
}
return NULL;
}
enum cache_pending_action get_pragma_action(const char * value)
{
const char *pragma_value = "no-cache";
if (strcasecmp(value, pragma_value) == 0)
{
return REVALIDATE;
}
return UNDEFINED;
}
enum cache_pending_action tfe_cache_get_pending(const struct tfe_http_half *request, struct request_freshness* restrict)
{
enum cache_pending_action res = UNDEFINED;
const char *value = NULL;
memset(restrict,0,sizeof(struct request_freshness));
if(request->req_spec.method!=TFE_HTTP_METHOD_GET)
{
return FORBIDDEN;
}
if(NULL!=tfe_http_std_field_read(request, TFE_HTTP_CONT_RANGE) ||
NULL!=tfe_http_std_field_read(request, TFE_HTTP_AUTHORIZATION)||
NULL!=tfe_http_nonstd_field_read(request, "WWW-Authenticate"))
{
return FORBIDDEN;
}
value = tfe_http_std_field_read(request, TFE_HTTP_PRAGMA);
if (value != NULL)
{
res = get_pragma_action(value);
}
else
{
value = tfe_http_std_field_read(request, TFE_HTTP_CACHE_CONTROL);
if (value != NULL)
{
res = request_cache_control(value, restrict);
}
else
{
if (cache_verify(request))
{
res = REVALIDATE;
}
}
}
return res;
}
time_t read_GMT_time(const char* gmt_string)
{
time_t expire_rel_time;
struct tm expire_gmt_time;
memset(&expire_gmt_time, 0, sizeof(expire_gmt_time));
strptime(gmt_string, "%a, %d %b %Y %H:%M:%S GMT", &expire_gmt_time);
expire_rel_time = mktime(&expire_gmt_time);
return expire_rel_time;
}
bool is_standard_gmt_format(const char* value)
{
int str_len = strlen(value);
if(0==strcasecmp(value+str_len-3,"GMT"))
{
return true;
}
else
{
return false;
}
}
time_t get_response_s_maxage(const char* cache_ctl)
{
const char* s_maxage = NULL;
s_maxage = strstr(cache_ctl, "s-maxage");
if (s_maxage != NULL)
{
return get_time_value(s_maxage, "s-maxage");
}
else
{
return 0;
}
}
time_t get_response_maxage(const char* cache_ctl)
{
const char* max_age = NULL;
max_age = strstr(cache_ctl, "max-age");
if (max_age != NULL)
{
return get_time_value(max_age, "max-age");
}
else
{
return 0;
}
}
void get_response_freshness(const struct tfe_http_half *response, struct response_freshness* freshness)
{
time_t expire_rel_time = 0;
time_t cur_rel_time = 0;
struct tm cur_gmt_time;
const char* field_value = NULL;
field_value = tfe_http_std_field_read(response, TFE_HTTP_CACHE_CONTROL);
if (field_value != NULL)
{
freshness->timeout = get_response_s_maxage(field_value);
if (freshness->timeout == 0)
{
freshness->timeout = get_response_maxage(field_value);
}
}
else
{
field_value = tfe_http_std_field_read(response, TFE_HTTP_EXPIRES);
if (field_value != NULL && is_standard_gmt_format(field_value))
{
expire_rel_time = read_GMT_time(field_value);
const time_t cur_ct_time = time(NULL);
if (gmtime_r(&cur_ct_time, &cur_gmt_time) == NULL)
{
assert(0);
}
cur_rel_time = mktime(&cur_gmt_time);
freshness->timeout = expire_rel_time - cur_rel_time;
}
}
field_value = tfe_http_std_field_read(response, TFE_HTTP_DATE);
if (field_value != NULL)
{
if(is_standard_gmt_format(field_value))
{
freshness->date = read_GMT_time(field_value);;
}
}
field_value = tfe_http_std_field_read(response, TFE_HTTP_LAST_MODIFIED);
if (field_value != NULL && is_standard_gmt_format(field_value))
{
freshness->last_modified = read_GMT_time(field_value);;
}
}
enum cache_pending_action response_cache_control(const char* value)
{
const char *forbidden_vaule[] = {"no-store", "private"};
const char *verify_vaule[] = { "no-cache", "must-revalidate","proxy-revalidate" };
int i = 0;
for (i = 0; i < 2; i++)
{
if (strstr(value, forbidden_vaule[i]) != NULL)
{
return FORBIDDEN;
}
}
for (i = 0; i < 3; i++)
{
if (strstr(value, verify_vaule[i]) != NULL)
{
return REVALIDATE;
}
}
return ALLOWED;
}
enum cache_pending_action tfe_cache_put_pending(const struct tfe_http_half *response, struct response_freshness* freshness)
{
enum cache_pending_action res = UNDEFINED;
const char *value = NULL;
memset(freshness,0,sizeof(struct response_freshness));
if(response->resp_spec.resp_code!=TFE_HTTP_STATUS_OK
|| NULL!=tfe_http_std_field_read(response, TFE_HTTP_CONT_RANGE) //NOT upload response with content-range
|| NULL==response->resp_spec.content_length
|| NULL!=tfe_http_std_field_read(response, TFE_HTTP_AUTHORIZATION)
|| NULL!=tfe_http_nonstd_field_read(response, "WWW-Authenticate")
|| NULL!=tfe_http_std_field_read(response, TFE_HTTP_SET_COOKIE))
{
return FORBIDDEN;
}
value = tfe_http_std_field_read(response, TFE_HTTP_PRAGMA);
if (value != NULL)
{
res = get_pragma_action(value);
}
else
{
value = tfe_http_std_field_read(response, TFE_HTTP_CACHE_CONTROL);
if (value != NULL)
{
res = response_cache_control(value);
}
else
{
value = tfe_http_std_field_read(response, TFE_HTTP_EXPIRES);
if (value != NULL && 0!= read_GMT_time(value))
{
res = ALLOWED;
}
}
}
if (res == ALLOWED)
{
get_response_freshness(response, freshness);
}
return res;
}

422
cache/src/tango_cache_redis.cpp vendored
View File

@@ -1,422 +0,0 @@
#include <sys/types.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/time.h>
#include <time.h>
#include <string.h>
#include <hiredis-vip/hircluster.h>
#include <hiredis-vip/async.h>
#include <hiredis-vip/adapters/libevent.h>
#include <cjson/cJSON.h>
#include "tango_cache_transfer.h"
#include "tango_cache_tools.h"
#include "tango_cache_redis.h"
#define PARSE_JSON_RET_ERROR -1
#define PARSE_JSON_RET_TIMEOUT 0
#define PARSE_JSON_RET_SUCC 1
#define CACHE_REDIS_CONNECT_IDLE 0
#define CACHE_REDIS_CONNECTING 1
#define CACHE_REDIS_CONNECTED 2
#define CACHE_REDIS_DISCONNECTED 3
static void redis_asyn_disconnect_cb(const struct redisAsyncContext *ac, int status)
{
struct tango_cache_instance *instance = (struct tango_cache_instance *)redisAsyncGetConnectionData(ac);
if(status == REDIS_OK)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Redis disconnect %s:%d success.",
ac->c.tcp.host, ac->c.tcp.port);
}
else
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Redis disconnect %s:%d failed: %s.",
ac->c.tcp.host, ac->c.tcp.port, ac->errstr);
}
instance->redis_connecting = CACHE_REDIS_DISCONNECTED;
}
static void redis_asyn_connect_cb(const struct redisAsyncContext *ac, int status)
{
struct tango_cache_instance *instance = (struct tango_cache_instance *)redisAsyncGetConnectionData(ac);
if(status == REDIS_OK)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "RedisCluster connect %s:%d success.",
ac->c.tcp.host, ac->c.tcp.port);
instance->redis_connecting = CACHE_REDIS_CONNECTED;
}
else
{
instance->redis_connecting = CACHE_REDIS_CONNECT_IDLE;
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "RedisCluster connect %s:%d failed: %s.",
ac->c.tcp.host, ac->c.tcp.port, ac->errstr);
}
}
int redis_asyn_connect_init(struct tango_cache_instance *instance)
{
instance->redis_ac = redisClusterAsyncConnect(instance->param->redisaddrs, HIRCLUSTER_FLAG_ROUTE_USE_SLOTS);
if(instance->redis_ac == NULL)
{
return -1;
}
instance->redis_connecting = CACHE_REDIS_CONNECTING;
redisClusterLibeventAttach(instance->redis_ac, instance->evbase);
redisClusterAsyncSetConnectionData(instance->redis_ac, instance);
redisClusterAsyncSetConnectCallback(instance->redis_ac, redis_asyn_connect_cb);
redisClusterAsyncSetDisconnectCallback(instance->redis_ac, redis_asyn_disconnect_cb);
return 0;
}
static int parse_object_meta_json(struct tango_cache_ctx *ctx, const char *jcontent)
{
cJSON *root, *ptarget;
int ret = PARSE_JSON_RET_ERROR;
char usertag[2048];
size_t datalen;
if(NULL == (root=cJSON_Parse(jcontent)))
{
goto out_json;
}
if(NULL == (ptarget=cJSON_GetObjectItem(root, "Content-Length")))
{
goto out_json;
}
ctx->get.result.tlength = ptarget->valuedouble;
if(NULL==(ptarget=cJSON_GetObjectItem(root, "X-Amz-Meta-Lm")))
{
goto out_json;
}
ctx->get.last_modify = ptarget->valuedouble;
if(NULL==(ptarget=cJSON_GetObjectItem(root, "Expires")))
{
goto out_json;
}
ctx->get.expires = ptarget->valuedouble;
ctx->get.need_hdrs = RESPONSE_HDR_ALL;
if(!check_expires_fresh_header(ctx))
{
ret = PARSE_JSON_RET_TIMEOUT;
goto out_json;
}
if(NULL!=(ptarget=cJSON_GetObjectItem(root, "Headers")))
{
easy_string_savedata(&ctx->response, ptarget->valuestring, strlen(ptarget->valuestring));
}
if(NULL!=(ptarget=cJSON_GetObjectItem(root, "X-Amz-Meta-User")))
{
if((datalen = Base64_DecodeBlock((unsigned char*)ptarget->valuestring, strlen(ptarget->valuestring), (unsigned char*)usertag, 2048))>0)
{
easy_string_savedata(&ctx->get.response_tag, usertag, datalen);
}
}
cJSON_Delete(root);
return PARSE_JSON_RET_SUCC;
out_json:
cJSON_Delete(root);
return ret;
}
static void redis_hget_command_cb(struct redisClusterAsyncContext *ac, void *vreply, void *privdata)
{
redisReply *reply = (redisReply *)vreply;
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)privdata;
int ret;
ctx->instance->statistic.session_redis -= 1;
if(reply == NULL || reply->type!=REDIS_REPLY_ARRAY)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_EXEC);
if(reply!=NULL && reply->type==REDIS_REPLY_ERROR)
{
promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, reply->str);
}
else
{
promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx));
}
tango_cache_ctx_destroy(ctx);
return;
}
else if(reply->element[0]->type == REDIS_REPLY_NIL)
{
tango_cache_set_fail_state(ctx, CACHE_CACHE_MISS);
ctx->get.result.type = RESULT_TYPE_MISS;
promise_success(ctx->promise, &ctx->get.result);
promise_finish(ctx->promise);
tango_cache_ctx_destroy(ctx);
return;
}
switch(ctx->get.state)
{
case GET_STATE_REDIS_META:
ctx->get.result.location = (strcmp(reply->element[1]->str, "redis"))?OBJECT_IN_HOS:OBJECT_IN_REDIS;
break;
case GET_STATE_REDIS_ALL:
ctx->get.result.location = OBJECT_IN_REDIS;
break;
case GET_STATE_REDIS_TRY:
ctx->get.result.location = (strcmp(reply->element[1]->str, "redis"))?OBJECT_IN_HOS:OBJECT_IN_REDIS;
if(ctx->get.result.location == OBJECT_IN_HOS)
{
ctx->get.redis_redirect_minio_cb(ctx);
return;
}
ctx->locate = OBJECT_IN_REDIS;
break;
default: assert(0);break;
}
ret = parse_object_meta_json(ctx, reply->element[0]->str);
switch(ret)
{
case PARSE_JSON_RET_ERROR:
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_JSON);
promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx));
tango_cache_ctx_destroy(ctx);
break;
case PARSE_JSON_RET_TIMEOUT:
if(ctx->get.state == GET_STATE_DELETE && ctx->get.result.location==OBJECT_IN_HOS)
{
ctx->get.state = GET_STATE_END;
cache_delete_minio_object(ctx);
}
else
{
tango_cache_ctx_destroy(ctx);
}
break;
case PARSE_JSON_RET_SUCC:
fetch_header_over_biz(ctx);
if(ctx->get.state != GET_STATE_REDIS_META)
{
ctx->get.result.data_frag = reply->element[2]->str;
ctx->get.result.size = reply->element[2]->len;
ctx->get.result.type = RESULT_TYPE_BODY;
promise_success(ctx->promise, &ctx->get.result);
}
ctx->get.result.type = RESULT_TYPE_END;
promise_success(ctx->promise, &ctx->get.result);
promise_finish(ctx->promise);
tango_cache_ctx_destroy(ctx);
break;
default: assert(0);break;
}
}
int tango_cache_head_redis(struct tango_cache_ctx *ctx)
{
int ret = -1;
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx,
"HMGET %s OBJECT_META OBJECT_LOCATION", ctx->object_key);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
tango_cache_ctx_destroy(ctx);
}
else
{
ctx->instance->statistic.session_redis += 1;
ctx->get.state = GET_STATE_REDIS_META;
}
return ret;
}
int tango_cache_fetch_redis(struct tango_cache_ctx *ctx)
{
int ret = -1;
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx,
"HMGET %s OBJECT_META OBJECT_LOCATION OBJECT_BODY", ctx->object_key);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
tango_cache_ctx_destroy(ctx);
}
else
{
ctx->instance->statistic.session_redis += 1;
ctx->get.state = GET_STATE_REDIS_ALL;
}
return ret;
}
int tango_cache_try_fetch_redis(struct tango_cache_ctx *ctx)
{
int ret = -1;
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx,
"HMGET %s OBJECT_META OBJECT_LOCATION OBJECT_BODY", ctx->object_key);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
tango_cache_ctx_destroy(ctx);
}
else
{
ctx->instance->statistic.session_redis += 1;
ctx->get.state = GET_STATE_REDIS_TRY;
}
return ret;
}
static void redis_hset_command_cb(struct redisClusterAsyncContext *ac, void *vreply, void *privdata)
{
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)privdata;
redisReply *reply = (redisReply *)vreply;
int ret;
ctx->instance->statistic.session_redis -= 1;
if(reply == NULL || reply->type==REDIS_REPLY_ERROR || ac->err)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_EXEC);
}
if(ctx->fail_state)
{
if(ctx->put.state==PUT_STATE_REDIS_META || ctx->put.state==PUT_STATE_REDIS_SETEX) //<2F><><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ص<EFBFBD>
{
ctx->put.state = PUT_STATE_END;
return;
}
tango_cache_ctx_destroy(ctx, true);
return;
}
switch(ctx->put.state)
{
case PUT_STATE_REDIS_META:
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx,
"EXPIRE %s %u", ctx->object_key, ctx->put.object_ttl);
if(ret!=REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
ctx->put.state = PUT_STATE_END;
}
else
{
ctx->instance->statistic.session_redis += 1;
ctx->put.state = PUT_STATE_REDIS_SETEX;
}
break;
case PUT_STATE_REDIS_EXPIRE:
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx,
"EXPIRE %s %u", ctx->object_key, ctx->put.object_ttl);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
tango_cache_ctx_destroy(ctx, true);
}
else
{
ctx->instance->statistic.session_redis += 1;
ctx->put.state = PUT_STATE_END;
}
break;
case PUT_STATE_REDIS_SETEX:
ctx->put.state = PUT_STATE_END; //<2F><><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>EXPIRE<52><45><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
break;
case PUT_STATE_END:
tango_cache_ctx_destroy(ctx, true);
break;
default: assert(0);break;
}
}
int redis_put_minio_object_meta(struct tango_cache_ctx *ctx, bool callback)
{
int ret_mset, ret_set;
char *meta;
meta = cJSON_PrintUnformatted(ctx->put.object_meta);
ret_mset = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx,
"HMSET %s OBJECT_LOCATION minio OBJECT_META %s MINIO_ADDR %s", ctx->object_key, meta, ctx->hostaddr);
ret_set = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx,
"SET http://%s/%s 1 EX %u", ctx->hostaddr, ctx->object_key, ctx->put.object_ttl);
if(ret_mset==REDIS_OK && ret_set==REDIS_OK)
{
ctx->instance->statistic.session_redis += 2;
ctx->put.state = PUT_STATE_REDIS_META;
}
else
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
if(ret_mset==REDIS_OK)
{
ctx->instance->statistic.session_redis += 1;
ctx->put.state = PUT_STATE_REDIS_EXPIRE; //<2F><>ʱ<EFBFBD><CAB1>PUT<55><54><EFBFBD><EFBFBD>object<63>߼<EFBFBD>һ<EFBFBD><D2BB>
}
else if(ret_set==REDIS_OK)
{
ctx->instance->statistic.session_redis += 1;
ctx->put.state = PUT_STATE_END;
}
else
{
tango_cache_ctx_destroy(ctx, callback);
free(meta);
return -1;
}
}
free(meta);
return 0;
}
int redis_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback)
{
int ret;
char *meta;
ctx->instance->statistic.memory_used -= size;
meta = cJSON_PrintUnformatted(ctx->put.object_meta);
ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx,
"HMSET %s OBJECT_LOCATION redis OBJECT_META %s OBJECT_BODY %b", ctx->object_key, meta, data, size);
if(ret != REDIS_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT);
tango_cache_ctx_destroy(ctx, callback);
}
else
{
ctx->instance->statistic.session_redis += 1;
ctx->put.state = PUT_STATE_REDIS_EXPIRE;
}
if(way == PUT_MEM_FREE)
{
free((void *)data);
}
free(meta);
return ret;
}
int redis_put_complete_part_evbuf(struct tango_cache_ctx *ctx, size_t object_size, bool callback)
{
char *data;
size_t size;
data = (char *)malloc(object_size);
size = evbuffer_remove(ctx->put.evbuf, data, object_size);
if(size != object_size)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_EVBUFFER);
tango_cache_ctx_destroy(ctx, callback);
free(data);
return CACHE_ERR_EVBUFFER;
}
return redis_put_complete_part_data(ctx, PUT_MEM_FREE, data, object_size, callback);
}

21
cache/src/tango_cache_redis.h vendored
View File

@@ -1,21 +0,0 @@
#ifndef __TANGO_CACHE_REDIS_H__
#define __TANGO_CACHE_REDIS_H__
#include <event2/event.h>
#include <event.h>
#include "tango_cache_client_in.h"
int tango_cache_head_redis(struct tango_cache_ctx *ctx);
int redis_asyn_connect_init(struct tango_cache_instance *instance);
int tango_cache_fetch_redis(struct tango_cache_ctx *ctx);
int tango_cache_try_fetch_redis(struct tango_cache_ctx *ctx);
int redis_put_minio_object_meta(struct tango_cache_ctx *ctx, bool callback);
int redis_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback);
int redis_put_complete_part_evbuf(struct tango_cache_ctx *ctx, size_t object_size, bool callback);
#endif

258
cache/src/tango_cache_tools.cpp vendored
View File

@@ -1,258 +0,0 @@
#include <sys/time.h>
#include <time.h>
#include <string.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include "tango_cache_tools.h"
static const char data_bin2ascii[65] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
#define conv_bin2ascii(a) (data_bin2ascii[(a)&0x3f])
#define B64_EOLN 0xF0
#define B64_CR 0xF1
#define B64_EOF 0xF2
#define B64_WS 0xE0
#define B64_ERROR 0xFF
#define B64_NOT_BASE64(a) (((a)|0x13) == 0xF3)
#define B64_BASE64(a) !B64_NOT_BASE64(a)
static const unsigned char data_ascii2bin[128] = {
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xE0, 0xF0, 0xFF, 0xFF, 0xF1, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0x3E, 0xFF, 0xF2, 0xFF, 0x3F,
0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B,
0x3C, 0x3D, 0xFF, 0xFF, 0xFF, 0x00, 0xFF, 0xFF,
0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E,
0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
0x17, 0x18, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20,
0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28,
0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30,
0x31, 0x32, 0x33, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
};
//<2F>мDz<D0BC><C7B2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>˵<EFBFBD><CBB5><EFBFBD>: conv_ascii2bin(x++); <20><>Ϊ<EFBFBD><CEAA>++<2B><><EFBFBD><EFBFBD>
#define conv_ascii2bin(aa) (((aa) & 0x80)?(0xFF):data_ascii2bin[(aa)])
/*********************************************************************
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƣ<EFBFBD>Base64_EncodeBlock
<EFBFBD><EFBFBD><EFBFBD>ܼ<EFBFBD><EFBFBD>飺<EFBFBD><EFBFBD>һ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>BASE64<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>б<EFBFBD><EFBFBD><EFBFBD>
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>in<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD><EFBFBD>
inl<6E><6C>in<69>ij<EFBFBD><C4B3><EFBFBD>
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>out<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>洢<EFBFBD>Ļ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
<EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ij<EFBFBD><EFBFBD><EFBFBD>
*********************************************************************/
int Base64_EncodeBlock(const unsigned char *in, int inl, unsigned char *out)
{
int i, ret = 0;
unsigned long l;
for (i = inl; i > 0; i -= 3) {
if (i >= 3) {
l = (((unsigned long)in[0]) << 16L) |
(((unsigned long)in[1]) << 8L) | in[2];
*(out++) = conv_bin2ascii(l >> 18L);
*(out++) = conv_bin2ascii(l >> 12L);
*(out++) = conv_bin2ascii(l >> 6L);
*(out++) = conv_bin2ascii(l);
} else {
l = ((unsigned long)in[0]) << 16L;
if (i == 2)
l |= ((unsigned long)in[1] << 8L);
*(out++) = conv_bin2ascii(l >> 18L);
*(out++) = conv_bin2ascii(l >> 12L);
*(out++) = (i == 1) ? '=' : conv_bin2ascii(l >> 6L);
*(out++) = '=';
}
ret += 4;
in += 3;
}
*out = '\0';
return (ret);
}
/*********************************************************************
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƣ<EFBFBD>Base64_DecodeBlock
<EFBFBD><EFBFBD><EFBFBD>ܼ<EFBFBD><EFBFBD>飺<EFBFBD><EFBFBD>һ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>BASE64<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>н<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Զ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>β<EFBFBD><EFBFBD>BASE64<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ַ<EFBFBD>
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>in<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD><EFBFBD>
inl<6E><6C>in<69>ij<EFBFBD><C4B3><EFBFBD>
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>out<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>洢<EFBFBD>Ļ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
<09><><EFBFBD>뱣֤<EBB1A3><D6A4><EFBFBD>㹻<EFBFBD>Ŀռ䣬һ<E4A3AC><D2BB><EFBFBD>ﵽ@inl<6E><6C>С<EFBFBD><D0A1><EFBFBD>ɣ<EFBFBD>
<EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵ<EFBFBD><EFBFBD><0<><30>ʧ<EFBFBD>ܣ<EFBFBD>>=0<><30><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ij<EFBFBD><C4B3><EFBFBD>
*********************************************************************/
int Base64_DecodeBlock(const unsigned char *in, int inl, unsigned char *out, int outsize)
{
int i, ret = 0;
unsigned char a, b, c, d;
unsigned long l;
/* ignore not-base64-encoded charactor. */
while ((conv_ascii2bin(*in) == B64_WS) && (inl > 0))
{
in++;
inl--;
}
while ((inl > 3) && (B64_NOT_BASE64(conv_ascii2bin(in[inl - 1]))))
inl--;
if (inl % 4 != 0)
return -1;
if(outsize < (inl*3)/4)
return -2;
for (i = 0; i < inl; i += 4)
{
a = conv_ascii2bin(*(in));
b = conv_ascii2bin(*(in+1));
c = conv_ascii2bin(*(in+2));
d = conv_ascii2bin(*(in+3));
if ((a & 0x80) || (b & 0x80) || (c & 0x80) || (d & 0x80))
return (-1);
l = ((((unsigned long)a) << 18L) | (((unsigned long)b) << 12L) |
(((unsigned long)c) << 6L) | (((unsigned long)d)));
*(out++) = (unsigned char)(l >> 16L) & 0xff;
*(out++) = (unsigned char)(l >> 8L) & 0xff;
*(out++) = (unsigned char)(l) & 0xff;
ret += 3;
in+=4;
}
for(i = inl; i > 0; i -= 4)
{
if(*(in-3) == '=')
{
in -= 4;
ret -= 3;
continue;
}
while(*(--in) == '=')
ret -= 1;
break;
}
return ret;
}
//<2F><>֪<EFBFBD><D6AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>£<EFBFBD><C2A3>Ƚ<EFBFBD><C8BD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD><EFBFBD><EFBFBD>
int strcmp_one_word_mesa_equal_len(const char *s1_lowercase, const char *s1_uppercase, const char *s2, size_t len)
{
unsigned char *s1,*s12;
if (s2[len-1]-'a'>=0)
{
s1 = (unsigned char *)s1_lowercase;
s12= (unsigned char *)s1_uppercase;
}
else
{
s1 = (unsigned char *)s1_uppercase;
s12= (unsigned char *)s1_lowercase;
}
do {
if (*s1 == *s2 || *s12 == *s2)
{
++s1;
++s12;
++s2;
continue;
}
return 0;
} while (--len);
return 1;
}
int mkdir_according_path(const char * path)
{
char buffer[256];
const char *ps=path, *pc;
if(*ps == '/')
ps += 1;
while((pc = strchr(ps, '/')) != NULL)
{
while(*(pc+1) == '/')
pc++;
memcpy(buffer, path, pc - path);
buffer[pc-path] = '\0';
if(access(buffer, F_OK))
{
if(mkdir(buffer, 0777) && errno!=EEXIST)
{
return -1;
}
}
ps = pc + 1;
}
if(access(path, F_OK))
{
if(mkdir(path, 0777))
{
return -1;
}
}
return 0;
}
//<2F><>ʱ<EFBFBD><CAB1><EFBFBD>ַ<EFBFBD><D6B7><EFBFBD>ת<EFBFBD><D7AA>Ϊʱ<CEAA><CAB1><EFBFBD><EFBFBD>
time_t expires_hdr2timestamp(const char *expires_val, int len)
{
struct tm tm;
while(len > 0 && (*expires_val==' '||*expires_val=='\t'||*expires_val=='\r'||*expires_val=='\n'))
{
expires_val++;
len--;
}
if(len == 0)
{
return 0;
}
memset(&tm, 0, sizeof(struct tm));
if(strptime(expires_val, "%a, %d %b %Y %T", &tm) == NULL)
{
return 0;
}
return mktime(&tm);
}
//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ת<EFBFBD><D7AA>ΪGMTʱ<54><CAB1><EFBFBD>ַ<EFBFBD><D6B7><EFBFBD>
size_t expires_timestamp2hdr_str(time_t seconds, char *buffer, size_t size)
{
struct tm save;
return strftime(buffer, size, "Expires: %a, %d %b %Y %T GMT", gmtime_r(&seconds, &save));
}
//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ת<EFBFBD><D7AA>ΪGMTʱ<54><CAB1><EFBFBD><EFBFBD>
time_t get_gmtime_timestamp(time_t seconds)
{
struct tm *tm, save;
tm = gmtime_r(&seconds, &save);
return mktime(tm);
}

26
cache/src/tango_cache_tools.h vendored
View File

@@ -1,26 +0,0 @@
#ifndef __TANGO_CACHE_TOOLS_H__
#define __TANGO_CACHE_TOOLS_H__
#include <MESA/MESA_handle_logger.h>
#define MESA_HANDLE_RUNTIME_LOGV2(handle, lv, fmt, args...) \
MESA_handle_runtime_log((handle), (lv), "TANGO_CACHE", "%s:%d, " fmt, __FILE__, __LINE__, ##args)
#ifdef CACHE_DEBUG_SWITCH
#define DBG_CACHE(fmt, args...) do{printf("%s():%d, " fmt, __FUNCTION__, __LINE__, ##args);}while(0)
#else
#define DBG_CACHE(msg...)
#endif
int Base64_EncodeBlock(const unsigned char *in, int inl, unsigned char *out);
int Base64_DecodeBlock(const unsigned char *in, int inl, unsigned char *out, int outsize);
int strcmp_one_word_mesa_equal_len(const char *s1_lowercase, const char *s1_uppercase, const char *s2, size_t len);
int mkdir_according_path(const char * path);
time_t get_gmtime_timestamp(time_t seconds);
time_t expires_hdr2timestamp(const char *expires_val, int len);
size_t expires_timestamp2hdr_str(time_t seconds, char *buffer, size_t size);
#endif

986
cache/src/tango_cache_transfer.cpp vendored
View File

@@ -1,986 +0,0 @@
#include <sys/types.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/time.h>
#include <time.h>
#include <string.h>
#include <curl/curl.h>
#include "tango_cache_transfer.h"
#include "tango_cache_xml.h"
#include "tango_cache_tools.h"
#include "tango_cache_redis.h"
static inline void curl_set_common_options(CURL *curl, long transfer_timeout, char *errorbuf)
{
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 500L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //<2F><><EFBFBD>Է<EFBFBD><D4B7>ֶ<EFBFBD><D6B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ij<EFBFBD><C4B3><EFBFBD>ӽ<EFBFBD><D3BD>տ<EFBFBD>ס<EFBFBD><D7A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
//ctx->error="Operation too slow. Less than 1024 bytes/sec transferred the last 3 seconds"
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 5L);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache");
}
//response body<64>̻ܶ򲻹<CCBB><F2B2BBB9><EFBFBD>ʱ
size_t curl_response_any_cb(void *ptr, size_t size, size_t count, void *userp)
{
return size*count;
}
static size_t curl_put_multipart_header_cb(void *ptr, size_t size, size_t count, void *userp)
{
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
size_t totallen = size*count;
char *start = (char *)ptr, *end = start + totallen;
struct multipart_etag_list *etag;
if(!strncmp(start, "Etag:", totallen>5?5:totallen))
{
start += 5; end -= 1; totallen -= 5;
while(totallen>0 && (*start==' ')) {start++; totallen--;}
while(totallen>0 && (*end=='\r'||*end=='\n')) {end--; totallen--;}
if(totallen > 0)
{
etag = (struct multipart_etag_list *)malloc(sizeof(struct multipart_etag_list));
totallen = end - start + 1;
etag->etag = (char *)malloc(totallen + 1);
etag->part_number = ctx->put.part_index;
memcpy(etag->etag, start, totallen);
*(etag->etag + totallen) = '\0';
TAILQ_INSERT_TAIL(&ctx->put.etag_head, etag, node);
}
}
return size*count;
}
static size_t curl_put_once_send_cb(void *ptr, size_t size, size_t count, void *userp)
{
size_t len;
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
if(size==0 || count==0 || ctx->put.once_request.len>=ctx->put.once_request.size)
{
return 0; //<2F><>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
}
len = ctx->put.once_request.size - ctx->put.once_request.len; //ʣ<><CAA3><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ij<EFBFBD><C4B3><EFBFBD>
if(len > size * count)
{
len = size * count;
}
memcpy(ptr, ctx->put.once_request.buff + ctx->put.once_request.len, len);
ctx->put.once_request.len += len;
if(ctx->put.once_request.len >= ctx->put.once_request.size)
{
ctx->instance->statistic.memory_used -= ctx->put.once_request.size; //δʹ<CEB4><CAB9>cache buffer<65><72><EFBFBD>Լ<EFBFBD><D4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD><EFBFBD><EFBFBD>
easy_string_destroy(&ctx->put.once_request);
}
return len;
}
static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, void *userp)
{
size_t len, space=size*count, send_len;
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
if(size==0 || count==0 || ctx->put.upload_offset>=ctx->put.upload_length)
{
return 0;
}
len = ctx->put.upload_length - ctx->put.upload_offset;
if(len > space)
{
len = space;
}
send_len = evbuffer_remove(ctx->put.evbuf, ptr, len);
assert(send_len>0);
ctx->put.upload_offset += send_len;
ctx->instance->statistic.memory_used -= send_len;
return send_len;
}
//return value: <0:fail; =0: not exec; >0: OK
static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool full)
{
UNUSED CURLMcode rc;
char minio_url[256]={0}, buffer[256]={0};
if(NULL == (ctx->curl=curl_easy_init()))
{
return -1;
}
ctx->put.upload_offset = 0;
if(full)
{
snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key);
}
else
{
snprintf(minio_url, 256, "http://%s/%s?partNumber=%d&uploadId=%s", ctx->hostaddr, ctx->object_key, ++ctx->put.part_index, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
}
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
//token<65>ֶΣ<D6B6><CEA3><EFBFBD><EFBFBD><EFBFBD>hos<6F>洢<EFBFBD><E6B4A2>֤
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put.upload_length);
curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb);
curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
DBG_CACHE("state: %d, length: %lu, key: %s\n", ctx->put.state, ctx->put.upload_length, ctx->object_key);
return 1;
}
static size_t curl_response_body_save_cb(void *ptr, size_t size, size_t count, void *userp)
{
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
struct easy_string *estr = &ctx->response;
CURLcode code;
if(ctx->fail_state)
{
return size*count;
}
if(ctx->res_code == 0)
{
code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code);
if(code != CURLE_OK || ctx->res_code!=200L)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
return size*count;
}
}
easy_string_savedata(estr, (const char*)ptr, size*count);
return size*count;
}
int curl_get_minio_uploadID(struct tango_cache_ctx *ctx)
{
UNUSED CURLMcode rc;
char minio_url[256]={0}, buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
return -1;
}
snprintf(minio_url, 256, "http://%s/%s?uploads", ctx->hostaddr, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, 0); //Ĭ<><C4AC>ʹ<EFBFBD>ûص<C3BB><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>fread<61><64><EFBFBD><EFBFBD><EFBFBD>Է<EFBFBD><D4B7>ֹر<D6B9>Expectʱ<74>ᵼ<EFBFBD>¿<EFBFBD><C2BF><EFBFBD>curl_multi_socket_action
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
DBG_CACHE("state: %d, key: %s\n", ctx->put.state, ctx->object_key);
return 1;
}
int cache_delete_minio_object(struct tango_cache_ctx *ctx, bool call_back)
{
UNUSED CURLMcode rc;
char minio_url[256], buffer[256];
ctx->instance->statistic.del_recv_num += 1;
if(NULL == (ctx->curl=curl_easy_init()))
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
tango_cache_ctx_destroy(ctx, call_back); //<2F>ս<EFBFBD><D5BD><EFBFBD>
return -1;
}
snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
return 1;
}
//return value: true-<2D>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD><C2BC><EFBFBD>false-δ<><CEB4><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD>
bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx)
{
UNUSED CURLMcode rc;
char minio_url[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
return false;
}
snprintf(minio_url, 256, "http://%s/%s?uploadId=%s", ctx->hostaddr, ctx->object_key, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
return true;
}
//return value: true-<2D>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD><C2BC><EFBFBD>false-δ<><CEB4><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD>
bool cache_kick_combine_minio(struct tango_cache_ctx *ctx)
{
int len=0;
UNUSED CURLMcode rc;
char minio_url[256], buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
return false;
}
construct_complete_xml(ctx, &ctx->put.combine_xml, &len);
snprintf(minio_url, 256, "http://%s/%s?uploadId=%s", ctx->hostaddr, ctx->object_key, ctx->put.uploadID);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, len); //<2F><><EFBFBD><EFBFBD>Content-Length
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->put.combine_xml);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
if(ctx->headers != NULL)
{
curl_slist_free_all(ctx->headers);
ctx->headers = NULL;
}
ctx->headers = curl_slist_append(ctx->headers, "Content-Type: application/xml");
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
DBG_CACHE("state: %d, key: %s\n", ctx->put.state, ctx->object_key);
return true;
}
//return value: true-<2D>ɹ<EFBFBD><C9B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD><C2BC><EFBFBD>false-δ<><CEB4><EFBFBD><EFBFBD><EFBFBD>¼<EFBFBD>
bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block_len)
{
int ret = 1;
switch(ctx->put.state)
{
case PUT_STATE_START:
if(sessions_exceeds_limit(ctx->instance, OBJECT_IN_HOS))
{
tango_cache_set_fail_state(ctx, CACHE_OUTOF_SESSION);
return false;
}
ctx->put.state = PUT_STATE_WAIT_START;
ret = curl_get_minio_uploadID(ctx);
break;
case PUT_STATE_PART:
if(ctx->curl == NULL)
{
ctx->put.upload_length = block_len;
ret = http_put_bodypart_request_evbuf(ctx, false);
}
break;
default: break;//nothing to do
}
if(ret <= 0)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
return false;
}
return true;
}
//callbackֱ<6B><D6B1>ʧ<EFBFBD><CAA7><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD><EFBFBD>ûص<C3BB><D8B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>Բ<EFBFBD><D4B2><EFBFBD>Ҫ
static int http_put_complete_part_evbuf(struct tango_cache_ctx *ctx, bool callback)
{
int ret=-1;
ctx->put.state = PUT_STATE_END;
ctx->put.upload_length = evbuffer_get_length(ctx->put.evbuf);
if(ctx->put.upload_length > 0)
{
ret = http_put_bodypart_request_evbuf(ctx, true);
if(ret <= 0)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
tango_cache_ctx_destroy(ctx, callback);
}
}
else
{
tango_cache_ctx_destroy(ctx, callback);
}
return ret;
}
int do_tango_cache_update_end(struct tango_cache_ctx *ctx, bool callback)
{
DBG_CACHE("state: %d, key: %s, curl %s NULL\n", ctx->put.state, ctx->object_key, (ctx->curl==NULL)?"is":"is not");
ctx->put.close_state = true;//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>رգ<D8B1><D5A3>ڲ<EFBFBD>״̬<D7B4><CCAC><EFBFBD><EFBFBD>ת<EFBFBD><D7AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ٹر<D9B9>
if(ctx->fail_state)
{
tango_cache_ctx_destroy(ctx, callback);
return -1;
}
switch(ctx->put.state)
{
case PUT_STATE_START: //<2F><>ʱ<EFBFBD><CAB1>ͬ<EFBFBD><CDAC><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD>
if(sessions_exceeds_limit(ctx->instance, ctx->locate))
{
tango_cache_set_fail_state(ctx, CACHE_OUTOF_SESSION);
tango_cache_ctx_destroy(ctx, callback);
return -1;
}
if(ctx->locate == OBJECT_IN_HOS)
{
return http_put_complete_part_evbuf(ctx, callback);
}
else
{
return redis_put_complete_part_evbuf(ctx, ctx->put.object_size, callback);
}
break;
case PUT_STATE_PART:
if(ctx->curl == NULL)
{
ctx->put.upload_length = evbuffer_get_length(ctx->put.evbuf);
if(ctx->put.upload_length == 0)
{
if(cache_kick_combine_minio(ctx))
{
ctx->put.state = PUT_STATE_END;
}
else
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
tango_cache_ctx_destroy(ctx);
return -1;
}
}
else if(http_put_bodypart_request_evbuf(ctx, false) <= 0)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
if(cache_cancel_upload_minio(ctx))
{
ctx->put.state = PUT_STATE_CANCEL;
}
else
{
tango_cache_ctx_destroy(ctx);
return -1;
}
}
}
break;
case PUT_STATE_END: assert(0); //<2F>û<EFBFBD><C3BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>endʱ<64><CAB1><EFBFBD><EFBFBD><EFBFBD>ܴ<EFBFBD><DCB4>ڴ<EFBFBD>״̬
case PUT_STATE_WAIT_START: //<2F><>ʱδ<CAB1><CEB4>ȡ<EFBFBD><C8A1>uploadId<49><64><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>޷<EFBFBD><DEB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD>
default: break;
}
return 0;
}
void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
DBG_CACHE("state: %d, key: %s\n", ctx->put.state, ctx->object_key);
switch(ctx->put.state)
{
case PUT_STATE_WAIT_START:
if(res!=CURLE_OK||res_code!=200L|| ctx->fail_state || !parse_uploadID_xml(ctx->response.buff, ctx->response.len, &ctx->put.uploadID))
{
easy_string_destroy(&ctx->response);
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
if(ctx->put.close_state)
{
tango_cache_ctx_destroy(ctx);
}
}
else
{
easy_string_destroy(&ctx->response);
ctx->put.state = PUT_STATE_PART;
if(ctx->put.close_state)
{
do_tango_cache_update_end(ctx, true);
}
else
{
size_t upload_length = evbuffer_get_length(ctx->put.evbuf);
if(upload_length >= ctx->instance->param->upload_block_size)
{
cache_kick_upload_minio_multipart(ctx, upload_length);
}
}
}
break;
case PUT_STATE_PART:
if(res != CURLE_OK || res_code!=200L)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
}
if(ctx->fail_state)
{
if(cache_cancel_upload_minio(ctx))
{
ctx->put.state = PUT_STATE_CANCEL;
}
else if(ctx->put.close_state)
{
tango_cache_ctx_destroy(ctx);
}
}
else if(ctx->put.close_state)
{
do_tango_cache_update_end(ctx, true);
}
else
{
size_t upload_length = evbuffer_get_length(ctx->put.evbuf);
if(upload_length >= ctx->instance->param->upload_block_size)
{
cache_kick_upload_minio_multipart(ctx, upload_length);
}
}
break;
case PUT_STATE_CANCEL: //<2F>ȴ<EFBFBD><C8B4>ر<EFBFBD>
if(ctx->put.close_state)
{
tango_cache_ctx_destroy(ctx);
}
break;
case PUT_STATE_END:
if(res != CURLE_OK || res_code!=200L)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
}
if(ctx->instance->param->object_store_way!=CACHE_ALL_HOS && !ctx->fail_state)
{
redis_put_minio_object_meta(ctx, true);
}
else
{
tango_cache_ctx_destroy(ctx);
}
break;
default: break;
}
}
int http_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback)
{
UNUSED CURLMcode rc;
char minio_url[256], buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
tango_cache_ctx_destroy(ctx, callback);
if(way == PUT_MEM_FREE) free((void *)data);
ctx->instance->statistic.memory_used -= size;
return -1;
}
ctx->put.state = PUT_STATE_END;
snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
if(way == PUT_MEM_COPY)
{
ctx->put.once_request.buff = (char *)malloc(size);
memcpy(ctx->put.once_request.buff, data, size);
}
else
{
ctx->put.once_request.buff = (char *)data;
}
ctx->put.once_request.size = size;
ctx->put.once_request.len = 0;
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put.once_request.size);
curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_once_send_cb);
curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
return 0;
}
int do_tango_cache_upload_once_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback)
{
ctx->instance->statistic.put_recv_num += 1;
ctx->instance->statistic.memory_used += size;
ctx->instance->error_code = CACHE_OK;
if(ctx->locate == OBJECT_IN_HOS)
{
return http_put_complete_part_data(ctx, way, data, size, false);
}
else
{
return redis_put_complete_part_data(ctx, way, data, size, false);
}
}
int do_tango_cache_upload_once_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf, bool callback)
{
size_t size;
ctx->instance->statistic.put_recv_num += 1;
ctx->instance->error_code = CACHE_OK;
if(way == EVBUFFER_MOVE)
{
if(evbuffer_add_buffer(ctx->put.evbuf, evbuf))
{
tango_cache_set_fail_state(ctx, CACHE_ERR_EVBUFFER);
tango_cache_ctx_destroy(ctx, callback);
return -1;
}
}
else
{
if(evbuffer_add_buffer_reference(ctx->put.evbuf, evbuf))
{
tango_cache_set_fail_state(ctx, CACHE_ERR_EVBUFFER);
tango_cache_ctx_destroy(ctx, callback);
return -1;
}
}
size = evbuffer_get_length(ctx->put.evbuf);
ctx->instance->statistic.memory_used += size;
if(ctx->locate == OBJECT_IN_HOS)
{
return http_put_complete_part_evbuf(ctx, callback);
}
else
{
return redis_put_complete_part_evbuf(ctx, size, callback);
}
}
void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
if(res!=CURLE_OK || (res_code!=204L && res_code!=200L ))
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
}
tango_cache_ctx_destroy(ctx);
}
void tango_cache_curl_muldel_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
u_int32_t errnum=0;
if(res!=CURLE_OK || (res_code!=204L && res_code!=200L ))
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
ctx->del.fail_num = ctx->del.succ_num;
ctx->del.succ_num = 0;
}
else
{
if(!parse_multidelete_xml(ctx->response.buff, ctx->response.len, &errnum, ctx->error, CURL_ERROR_SIZE))
{
ctx->del.fail_num = ctx->del.succ_num;
ctx->del.succ_num = 0;
}
else
{
ctx->del.fail_num = errnum;
ctx->del.succ_num -= errnum;
}
if(ctx->del.fail_num > 0)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
}
}
tango_cache_ctx_destroy(ctx);
}
int do_tango_cache_multi_delete(struct tango_cache_ctx *ctx, bool callback)
{
UNUSED CURLMcode rc;
char minio_url[256], buffer[256];
ctx->instance->statistic.del_recv_num += ctx->del.succ_num;
ctx->instance->error_code = CACHE_OK;
if(NULL == (ctx->curl=curl_easy_init()))
{
tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY);
tango_cache_ctx_destroy(ctx, callback);
return -1;
}
snprintf(minio_url, 256, "http://%s/%s/?delete", ctx->hostaddr, ctx->instance->param->bucketname);
curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, ctx->response.size); //<2F><><EFBFBD><EFBFBD>Content-Length<74><68><EFBFBD><EFBFBD>CURLOPT_COPYPOSTFIELDS֮ǰ<D6AE><C7B0><EFBFBD><EFBFBD>
curl_easy_setopt(ctx->curl, CURLOPT_COPYPOSTFIELDS, ctx->response.buff);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
easy_string_destroy(&ctx->response);
return 0;
}
bool fetch_header_over_biz(struct tango_cache_ctx *ctx)
{
if(ctx->get.need_hdrs!=RESPONSE_HDR_ALL) //<2F><>Expiresʱ
{
tango_cache_set_fail_state(ctx, CACHE_ERR_INTERNAL);
ctx->get.state = GET_STATE_DELETE;
promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx));
return false;
}
if(ctx->get.response_tag.len > 0)
{
ctx->get.result.data_frag = ctx->get.response_tag.buff;
ctx->get.result.size = ctx->get.response_tag.len;
ctx->get.result.type = RESULT_TYPE_USERTAG;
promise_success(ctx->promise, &ctx->get.result);
easy_string_destroy(&ctx->get.response_tag);
}
if(ctx->response.len > 0)
{
ctx->get.result.data_frag = ctx->response.buff;
ctx->get.result.size = ctx->response.len;
ctx->get.result.type = RESULT_TYPE_HEADER;
promise_success(ctx->promise, &ctx->get.result);
easy_string_destroy(&ctx->response);
}
return true;
}
static size_t curl_get_response_body_cb(void *ptr, size_t size, size_t count, void *userp)
{
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
if(ctx->fail_state || ctx->get.state==GET_STATE_DELETE)
{
return size*count;
}
if(!fetch_header_over_biz(ctx))
{
return size*count;
}
ctx->get.result.data_frag = (const char *)ptr;
ctx->get.result.size = size * count;
ctx->get.result.type = RESULT_TYPE_BODY;
promise_success(ctx->promise, &ctx->get.result);
return size*count;
}
bool check_expires_fresh_header(struct tango_cache_ctx *ctx)
{
time_t now_gmt;
if(ctx->get.need_hdrs != RESPONSE_HDR_ALL)
return true;
now_gmt = get_gmtime_timestamp(time(NULL));
if(now_gmt > ctx->get.expires)
{
tango_cache_set_fail_state(ctx, CACHE_TIMEOUT);
ctx->get.state = GET_STATE_DELETE; //<2F><><EFBFBD><EFBFBD>ʧЧʱ<D0A7><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
ctx->get.result.type = RESULT_TYPE_MISS;
promise_success(ctx->promise, &ctx->get.result);
promise_finish(ctx->promise);
easy_string_destroy(&ctx->response);
return false;
}
if(ctx->get.last_modify+ctx->get.max_age > now_gmt || now_gmt+ctx->get.min_fresh>ctx->get.expires)
{
tango_cache_set_fail_state(ctx, CACHE_TIMEOUT);
ctx->get.result.type = RESULT_TYPE_MISS;
promise_success(ctx->promise, &ctx->get.result);
promise_finish(ctx->promise);
easy_string_destroy(&ctx->response);
return false;
}
return true;
}
static bool check_get_result_code(struct tango_cache_ctx *ctx, CURLcode code, long res_code)
{
if(code != CURLE_OK)
{
tango_cache_set_fail_state(ctx, CACHE_ERR_CURL);
promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx));
return false;
}
if(res_code != 200L)
{
if(res_code == 404L)
{
tango_cache_set_fail_state(ctx, CACHE_CACHE_MISS);
ctx->get.result.type = RESULT_TYPE_MISS;
promise_success(ctx->promise, &ctx->get.result);
promise_finish(ctx->promise);
}
else
{
tango_cache_set_fail_state(ctx, CACHE_ERR_INTERNAL);
promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx));
}
return false;
}
return true;
}
static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, void *userp)
{
struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp;
char *start=(char *)ptr, *pos_colon;
size_t raw_len = size*count, hdrlen=size*count;
char usertag[2048];
size_t datalen;
if(ctx->fail_state || ctx->get.state==GET_STATE_DELETE)
{
return raw_len;
}
if(ctx->res_code == 0) //<2F>״<EFBFBD>Ӧ<EFBFBD><D3A6>ʱ<EFBFBD>ȿ<EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>200
{
UNUSED CURLcode code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code);
if(!check_get_result_code(ctx, code, ctx->res_code))
{
return raw_len;
}
ctx->get.result.location = OBJECT_IN_HOS;
}
pos_colon = (char*)memchr(start, ':', raw_len);
if(pos_colon == NULL)
{
return raw_len;
}
datalen = pos_colon - start;
switch(datalen)
{
case 7:
if(strcmp_one_word_mesa_equal_len("expires", "EXPIRES", start, 7))
{
ctx->get.need_hdrs |= RESPONSE_HDR_EXPIRES;
ctx->get.expires = expires_hdr2timestamp(pos_colon + 1, raw_len - datalen - 1);
if(!check_expires_fresh_header(ctx))
{
return raw_len;
}
}
break;
case 13:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-lm", "X-AMZ-META-LM", start, 13))
{
ctx->get.need_hdrs |= RESPONSE_HDR_LAST_MOD;
sscanf(pos_colon+1, "%lu", &ctx->get.last_modify);
if(!check_expires_fresh_header(ctx))
{
return raw_len;
}
}
break;
case 15:
if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15))
{
if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0)
{
easy_string_savedata(&ctx->get.response_tag, usertag, hdrlen);
}
}
break;
case 14:
if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14))
{
sscanf(pos_colon+1, "%lu", &ctx->get.result.tlength);
}
break;
case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break;
default: break;
}
return raw_len;
}
void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code)
{
switch(ctx->get.state)
{
case GET_STATE_START:
if(!ctx->fail_state && check_get_result_code(ctx, res, res_code))
{
if(ctx->method!=CACHE_REQUEST_HEAD || fetch_header_over_biz(ctx)) //HEAD<41><44><EFBFBD>ֵ<EFBFBD><D6B5>ֶβ<D6B6>ȫ<EFBFBD>Ȳ<EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ޣ<EFBFBD>
{
ctx->get.result.type = RESULT_TYPE_END;
promise_success(ctx->promise, &ctx->get.result);
promise_finish(ctx->promise);
}
}
tango_cache_ctx_destroy(ctx);
break;
case GET_STATE_DELETE:
ctx->get.state = GET_STATE_END;
cache_delete_minio_object(ctx);
break;
case GET_STATE_END:
tango_cache_ctx_destroy(ctx);
break;
default: assert(0);break;
}
}
static int tango_cache_fetch_minio(struct tango_cache_ctx *ctx)
{
UNUSED CURLMcode rc;
char minio_url[256], buffer[256];
if(NULL == (ctx->curl=curl_easy_init()))
{
tango_cache_ctx_destroy(ctx);
return -1;
}
snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
if(ctx->method == CACHE_REQUEST_HEAD)
{
curl_easy_setopt(ctx->curl, CURLOPT_NOBODY, 1L);
}
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_get_response_body_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_get_response_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
sprintf(buffer, "token: %s", ctx->instance->param->cache_token);
ctx->headers = curl_slist_append(ctx->headers, buffer);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl);
assert(rc==CURLM_OK);
return 1;
}
static void redis_redirect_object2minio_cb(struct tango_cache_ctx *ctx)
{
struct promise *p = ctx->promise;
ctx->get.state = GET_STATE_START;
ctx->locate = OBJECT_IN_HOS;
if(ctx->instance->statistic.session_http>=ctx->instance->param->maximum_sessions)
{
tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY);
promise_failed(p, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx));
tango_cache_ctx_destroy(ctx);
}
else if(tango_cache_fetch_minio(ctx) != 1)
{
promise_failed(p, FUTURE_ERROR_CANCEL, "tango_cache_fetch_minio failed");
}
}
int do_tango_cache_fetch_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATION where_to_get)
{
ctx->instance->statistic.get_recv_num += 1;
switch(where_to_get)
{
case OBJECT_IN_HOS:
ctx->locate = OBJECT_IN_HOS;
return (tango_cache_fetch_minio(ctx)==1)?0:-2;
case OBJECT_IN_REDIS:
ctx->locate = OBJECT_IN_REDIS;
return tango_cache_fetch_redis(ctx);
default:
ctx->get.redis_redirect_minio_cb = redis_redirect_object2minio_cb;
return tango_cache_try_fetch_redis(ctx);
}
return 0;
}
int do_tango_cache_head_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATION where_to_head)
{
ctx->instance->statistic.get_recv_num += 1;
if(where_to_head == OBJECT_IN_REDIS)
{
return tango_cache_head_redis(ctx);
}
else
{
return (tango_cache_fetch_minio(ctx)==1)?0:-1;
}
}

31
cache/src/tango_cache_transfer.h vendored
View File

@@ -1,31 +0,0 @@
#ifndef __TANGO_CACHE_UPLOAD_H__
#define __TANGO_CACHE_UPLOAD_H__
#include <sys/queue.h>
#include <curl/curl.h>
#include "tango_cache_client_in.h"
bool check_expires_fresh_header(struct tango_cache_ctx *ctx);
bool fetch_header_over_biz(struct tango_cache_ctx *ctx);
void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
void tango_cache_curl_muldel_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code);
int cache_delete_minio_object(struct tango_cache_ctx *ctx, bool call_back=false);
int do_tango_cache_multi_delete(struct tango_cache_ctx *ctx, bool callback=false);
bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx);
bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block_len);
int do_tango_cache_update_end(struct tango_cache_ctx *ctx, bool callback);
int do_tango_cache_upload_once_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool call_back=false);
int do_tango_cache_upload_once_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf, bool call_back=false);
int do_tango_cache_head_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATION where_to_head);
int do_tango_cache_fetch_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATION where_to_get);
#endif

172
cache/src/tango_cache_xml.cpp vendored
View File

@@ -1,172 +0,0 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <libxml/parser.h>
#include "tango_cache_xml.h"
bool parse_uploadID_xml(const char *content, int len, char **uploadID)
{
xmlDoc *pdoc;
xmlNode *pcur;
if((pdoc = xmlParseMemory(content, len)) == NULL)
{
return false;
}
if((pcur = xmlDocGetRootElement(pdoc)) == NULL)
{
xmlFreeDoc(pdoc);
return false;
}
while(pcur->type != XML_ELEMENT_NODE)
pcur = pcur->next;
if(xmlStrcmp(pcur->name, (const xmlChar *)"InitiateMultipartUploadResult"))
{
xmlFreeDoc(pdoc);
return false;
}
pcur = pcur->children;
while(pcur != NULL)
{
if(pcur->type != XML_ELEMENT_NODE || xmlStrcmp(pcur->name, (const xmlChar *)"UploadId"))
{
pcur = pcur->next;
continue;
}
*uploadID = (char *)xmlNodeGetContent(pcur);
xmlFreeDoc(pdoc);
return true;
}
xmlFreeDoc(pdoc);
return false;
}
void construct_complete_xml(struct tango_cache_ctx *ctx, char **xml, int *len)
{
struct multipart_etag_list *etag;
xmlDoc *pdoc;
xmlNode *root, *child;
char number[20];
pdoc = xmlNewDoc((const xmlChar *)"1.0");
root = xmlNewNode(NULL, (const xmlChar *)"CompleteMultipartUpload");
/*Big data deletion of this field parsing, shielding this field**/
//xmlNewProp(root, (const xmlChar *)"xmlns",(const xmlChar *)"http://s3.amazonaws.com/doc/2006-03-01/");
xmlDocSetRootElement(pdoc, root);
TAILQ_FOREACH(etag, &ctx->put.etag_head, node)
{
sprintf(number, "%u", etag->part_number);
child = xmlNewChild(root, NULL, (const xmlChar*)"Part", NULL);
xmlNewChild(child, NULL, (const xmlChar*)"ETag", (const xmlChar*)etag->etag);
xmlNewChild(child, NULL, (const xmlChar*)"PartNumber", (const xmlChar*)number);
}
xmlDocDumpFormatMemory(pdoc, (xmlChar **)xml, len, 1);
xmlFreeDoc(pdoc);
}
static void fill_multidelete_xml_errcode(xmlNode *error, char *out, int size)
{
xmlChar *errcode;
xmlNode *child = error->children;
while(child != NULL)
{
if(child->type != XML_ELEMENT_NODE || xmlStrcmp(child->name, (const xmlChar *)"Message"))
{
child = child->next;
continue;
}
errcode = xmlNodeGetContent(child);
snprintf(out, size, "%s", (char *)errcode);
xmlFree(errcode);
break;
}
}
bool parse_multidelete_xml(const char *xml, int len, u_int32_t *errnum, char *errstr, int size)
{
xmlDoc *pdoc;
xmlNode *pcur;
int errornum=0;
if((pdoc = xmlParseMemory(xml, len)) == NULL)
{
return false;
}
if((pcur = xmlDocGetRootElement(pdoc)) == NULL)
{
xmlFreeDoc(pdoc);
return false;
}
while(pcur->type != XML_ELEMENT_NODE)
pcur = pcur->next;
if(xmlStrcmp(pcur->name, (const xmlChar *)"DeleteResult"))
{
xmlFreeDoc(pdoc);
return false;
}
pcur = pcur->children;
while(pcur != NULL)
{
if(pcur->type != XML_ELEMENT_NODE || xmlStrcmp(pcur->name, (const xmlChar *)"Error"))
{
pcur = pcur->next;
continue;
}
if(errornum == 0)
{
fill_multidelete_xml_errcode(pcur, errstr, size);
}
errornum++;
pcur = pcur->next;
}
*errnum = errornum;
xmlFreeDoc(pdoc);
return true;
}
void construct_multiple_delete_xml(const char *bucket, char *key[], u_int32_t num, int is_hash, char **xml, size_t *len)
{
xmlDoc *pdoc;
xmlNode *root, *child;
int xmllen;
pdoc = xmlNewDoc((const xmlChar *)"1.0");
root = xmlNewNode(NULL, (const xmlChar *)"Delete");
xmlDocSetRootElement(pdoc, root);
xmlNewChild(root, NULL, (const xmlChar*)"Quiet", (const xmlChar*)"true");
if(is_hash)
{
char hashkey[72], sha256[72];
for(u_int32_t i=0; i<num; i++)
{
child = xmlNewChild(root, NULL, (const xmlChar*)"Object", NULL);
caculate_sha256(key[i], strlen(key[i]), sha256, 72);
snprintf(hashkey, 256, "%c%c_%c%c_%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4);
xmlNewChild(child, NULL, (const xmlChar*)"Key", (const xmlChar*)hashkey);
}
}
else
{
for(u_int32_t i=0; i<num; i++)
{
child = xmlNewChild(root, NULL, (const xmlChar*)"Object", NULL);
xmlNewChild(child, NULL, (const xmlChar*)"Key", (const xmlChar*)key[i]);
}
}
xmlDocDumpFormatMemoryEnc(pdoc, (xmlChar **)xml, &xmllen, "UTF-8", 1);
xmlFreeDoc(pdoc);
*len = xmllen;
}

12
cache/src/tango_cache_xml.h vendored
View File

@@ -1,12 +0,0 @@
#ifndef __TANGO_CACHE_XML_H__
#define __TANGO_CACHE_XML_H__
#include "tango_cache_client_in.h"
bool parse_uploadID_xml(const char *content, int len, char **uploadID);
void construct_complete_xml(struct tango_cache_ctx *ctx, char **xml, int *len);
void construct_multiple_delete_xml(const char *bucket, char *key[], u_int32_t num, int is_hash, char **xml, size_t *len);
bool parse_multidelete_xml(const char *xml, int len, u_int32_t *errnum, char *errstr, int size);
#endif

24
cache/test/CMakeLists.txt vendored
View File

@@ -1,24 +0,0 @@
add_definitions(-fPIC -Wall -g)
add_executable (cache_evbase_test cache_evbase_test.cpp)
target_include_directories(cache_evbase_test PRIVATE ../include)
target_link_libraries(cache_evbase_test common)
target_link_libraries(cache_evbase_test tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (cache_evbase_test MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread z)
add_executable (cache_evbase_benchmark cache_evbase_benchmark.cpp)
target_link_libraries(cache_evbase_benchmark tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (cache_evbase_benchmark MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread z)
#add_executable (cache_evbase_test_threads cache_evbase_test_threads.cpp)
#target_link_libraries(cache_evbase_test_threads tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
#target_link_libraries (cache_evbase_test_threads MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread z)
add_executable (tango_cache_test tango_cache_test.cpp)
target_link_libraries(tango_cache_test tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (tango_cache_test MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread)
#INSTALL (TARGETS cache_evbase_test cache_evbase_test_threads tango_cache_test cache_evbase_benchmark DESTINATION bin)
INSTALL (TARGETS cache_evbase_test tango_cache_test cache_evbase_benchmark DESTINATION bin)
INSTALL (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pangu_tg_cahce.conf DESTINATION bin)
INSTALL (FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmd.txt DESTINATION bin)

39
cache/test/Makefile vendored
View File

@@ -1,39 +0,0 @@
CCC=g++
INC_PATH=-I../include -I../include/libevent2
CFLAGS=-Wall -g $(INC_PATH)
LIBS = -lMESA_handle_logger -lMESA_prof_load -lWiredLB
LIBS += -lssl -lcrypto
LIBS += ../lib/libtango_cache_client.a
LIBS += ./lib/libcurl.a ./lib/libevent.a ./lib/libxml2.a ./lib/libhiredis_vip.a
OBJS = tango_cache_test.o
OBJS_EVBASE=cache_evbase_test.o
OBJS_EVBASE_BENCHMARK=cache_evbase_benchmark.o
TARGET_EXE=tango_cache_test
TARGET_EXE_EVBASE=cache_evbase_test
TARGET_EXE_EVBASE_BENCHMARK=cache_evbase_benchmark
ALL:$(TARGET_EXE) $(TARGET_EXE_EVBASE) $(TARGET_EXE_EVBASE_BENCHMARK)
$(TARGET_EXE):$(OBJS)
$(CCC) $(LDFLAGS) $^ -o $@ $(LIBS)
$(TARGET_EXE_EVBASE):$(OBJS_EVBASE)
$(CCC) $(LDFLAGS) $^ -o $@ $(LIBS) -lpthread
$(TARGET_EXE_EVBASE_BENCHMARK):$(OBJS_EVBASE_BENCHMARK)
$(CCC) $(LDFLAGS) $^ -o $@ $(LIBS) -lpthread
.c.o:
$(CCC) $(CFLAGS) -c $<
.cpp.o:
$(CCC) $(CFLAGS) -c $<
-include $(DEPS)
clean:
rm -rf $(OBJS) $(TARGET_EXE) $(OBJS_EVBASE) $(TARGET_EXE_EVBASE) $(OBJS_EVBASE_BENCHMARK) $(TARGET_EXE_EVBASE_BENCHMARK)

16
cache/test/README.txt vendored
View File

@@ -1,16 +0,0 @@
1<EFBFBD><EFBFBD>tango_cache_test.c<><63><EFBFBD>칷ʹ<ECB9B7>õ<EFBFBD>demo<6D><6F><EFBFBD><EFBFBD>Ҫ<EFBFBD>ⲿ<EFBFBD><E2B2BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>event_base<73><65>ִ<EFBFBD>з<EFBFBD><D0B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>£<EFBFBD>
<09><><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>նˣ<D5B6>ִ<EFBFBD><D6B4>./tango_cache_test<73><74>
<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>նˣ<D5B6>ִ<EFBFBD>У<EFBFBD>
<09><>ȡ<EFBFBD><C8A1>echo GET:tcpdump_all_1.pcap > cache.fifo
<09><>ʽ<EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD>
echo PUT:tcpdump_all_1.pcap > cache.fifo
<09><><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD>
echo PUTONCE:tcpdump_all_1.pcap > cache.fifo
2<EFBFBD><EFBFBD>cache_evbase_test.c<>Ƕ<EFBFBD><C7B6>칷cache<68>ķ<EFBFBD>װ<EFBFBD><D7B0><EFBFBD>û<EFBFBD><C3BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Լ<EFBFBD><D4BC><EFBFBD><EFBFBD><EFBFBD>event_base<73><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڲ<EFBFBD><DAB2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ִ<EFBFBD>з<EFBFBD><D0B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>£<EFBFBD>
<09><>ȡ<EFBFBD><C8A1>./cache_evbase_test GET:tcpdump_all_1.pcap
<09><>ʽ<EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD>./cache_evbase_test PUT:tcpdump_all_1.pcap
<09><><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4><EFBFBD>./cache_evbase_test PUTONCE:tcpdump_all_1.pcap
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ж<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>tcpdump_all_1.pcap<61><70><EFBFBD>Եģ<D4B5>ʵ<EFBFBD><CAB5>Ӧ<EFBFBD><D3A6>ʱ<EFBFBD>滻<EFBFBD>ɾ<EFBFBD><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>

444
cache/test/cache_evbase_benchmark.cpp vendored
View File

@@ -1,444 +0,0 @@
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/prctl.h>
#include <sys/un.h>
#include <unistd.h>
#include <sys/poll.h>
#include <curl/curl.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
#include <sys/cdefs.h>
#include <MESA/MESA_handle_logger.h>
#include <pthread.h>
#include <sys/prctl.h>
#include "object_store_client.h"
#define METHOD_GET 1
#define METHOD_PUT 2
#define METHOD_HEAD 3
#define METHOD_PUTONCEEV 4
#define METHOD_PUTONCE 5
#define METHOD_DEL 6
struct object_store_instance *instance_asyn;
struct filecontent
{
char *buf;
size_t len;
};
struct future_pdata
{
struct future * future;
FILE *fp;
char filename[256];
};
void get_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = cache_evbase_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
//char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
//memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
//buffer[res->size] = '\0';
//printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
//fwrite(res->data_frag, res->size, 1, pdata->fp);
break;
case RESULT_TYPE_MISS:
//printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
//if(res->type != RESULT_TYPE_MISS)
// printf("get cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
//fclose(pdata->fp);
free(pdata);
break;
default:break;
}
}
void get_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
future_destroy(pdata->future);
free(pdata);
//printf("GET fail: %s\n", what);
}
void head_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = cache_evbase_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
assert(0);
break;
case RESULT_TYPE_MISS:
printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
if(res->type != RESULT_TYPE_MISS)
printf("HEAD cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
free(pdata);
break;
default:break;
}
}
void head_future_failed(enum e_future_error err, const char * what, void * user)
{
printf("HEAD fail: %s\n", what);
}
void put_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
//printf("PUT %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void put_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
//printf("PUT %s fail: %s\n", what, pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void del_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void del_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s fail: %s\n", pdata->filename, what);
future_destroy(pdata->future);
free(pdata);
}
char * get_file_content(const char *filename, size_t *filelen_out)
{
char *buffer;
FILE *fp;
size_t filelen = 0;
struct stat filestat;
int readlen;
fp = fopen(filename, "rb");
if(fp == NULL)
{
printf("fopen file %s failed.\n", filename);
return NULL;
}
if(fstat(fileno(fp), &filestat))
{
printf("fstat %s failed.\n", filename);
return NULL;
}
buffer = (char *)malloc(filestat.st_size);
while(filelen < (size_t)filestat.st_size)
{
readlen = fread(buffer + filelen, 1, filestat.st_size - filelen, fp);
if(readlen < 0)
{
printf("read error: %s\n", strerror(errno));
return NULL;
}
filelen += readlen;
}
fclose(fp);
*filelen_out = filestat.st_size;
return buffer;
}
struct cache_statistics g_out_last;
void timer_cb(evutil_socket_t fd, short what, void *arg)
{
struct timeval tv;
struct cache_statistics out_now;
struct cache_statistics out;
tv.tv_sec = 10;
tv.tv_usec = 0;
/*static int num=0;
num++;
if(ctx_global!=NULL && num>5)
{
tango_cache_update_end(ctx_global);
ctx_global = NULL;
}*/
object_store_get_statistics(instance_asyn, &out_now);
out.del_error_num = out_now.del_error_num - g_out_last.del_error_num;
out.del_recv_num = out_now.del_recv_num - g_out_last.del_recv_num;
out.del_succ_num = out_now.del_succ_num - g_out_last.del_succ_num;
out.get_err_http = out_now.get_err_http - g_out_last.get_err_http;
out.get_err_redis = out_now.get_err_redis - g_out_last.get_err_redis;
out.get_miss_num = out_now.get_miss_num - g_out_last.get_miss_num;
out.get_recv_num = out_now.get_recv_num - g_out_last.get_recv_num;
out.get_succ_http = out_now.get_succ_http - g_out_last.get_succ_http;
out.get_succ_redis = out_now.get_succ_redis - g_out_last.get_succ_redis;
out.put_err_http = out_now.put_err_http - g_out_last.put_err_http;
out.put_err_redis = out_now.put_err_redis - g_out_last.put_err_redis;
out.put_recv_num = out_now.put_recv_num - g_out_last.put_recv_num;
out.put_succ_http = out_now.put_succ_http - g_out_last.put_succ_http;
out.put_succ_redis = out_now.put_succ_redis - g_out_last.put_succ_redis;
out.session_http = out_now.session_http;
out.session_redis = out_now.session_redis;
out.memory_used = out_now.memory_used;
out.totaldrop_num = out_now.totaldrop_num - g_out_last.totaldrop_num;
printf("-------------------------------------------------------------------------------------------\n"
"get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n"
"put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n"
"del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num,
out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis,
out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used);
g_out_last = out_now;
event_add((struct event *)arg, &tv);
}
struct filecontentcmd
{
int method;
int threads;
int total_num;
int sess_limit;
char file[256];
};
static void* thread_transfer_cmd(void *arg)
{
int index=0;
char filename_in[256];
struct tango_cache_meta_put putmeta;
struct tango_cache_meta_get getmeta;
struct future_pdata *pdata;
struct cache_evbase_ctx *ctx;
struct filecontent filecont;
size_t remain_len;
struct cache_statistics out;
struct evbuffer *evbuf;
struct filecontentcmd *filecmd = (struct filecontentcmd *)arg;
prctl(PR_SET_NAME, "transfer_cmd");
memset(&putmeta, 0, sizeof(struct tango_cache_meta_put));
memset(&getmeta, 0, sizeof(struct tango_cache_meta_get));
putmeta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype";
putmeta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip";
putmeta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
putmeta.usertag_len = strlen(putmeta.usertag);
filecont.buf = get_file_content(filecmd->file, &filecont.len);
assert(filecont.buf != NULL);
while(1)
{
object_store_get_statistics(instance_asyn, &out);
if(out.session_http >= filecmd->sess_limit || out.session_redis>=filecmd->sess_limit)
{
usleep(1000);
continue;
}
switch(filecmd->method)
{
case METHOD_GET:
sprintf(filename_in, "%s_%u", filecmd->file, index);
getmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata);
object_store_fetch_object(instance_asyn, pdata->future, &getmeta, OBJECT_IN_UNKNOWN);
break;
case METHOD_PUT:
remain_len = filecont.len;
sprintf(filename_in, "%s_%u", filecmd->file, index);
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_put", put_future_success, put_future_failed, (void *)pdata);
ctx = object_store_update_start(instance_asyn, pdata->future, &putmeta);
if(ctx == NULL)
{
future_destroy(pdata->future);
free(pdata);
continue;
}
while(remain_len >= 1024)
{
object_store_update_frag_data(ctx, PUT_MEM_COPY, filecont.buf+(filecont.len-remain_len), 1024);
remain_len -= 1024;
}
if(remain_len > 0)
{
object_store_update_frag_data(ctx, PUT_MEM_COPY, filecont.buf+(filecont.len-remain_len), remain_len);
}
if(object_store_update_end(ctx, pdata->filename, 256))
{
future_destroy(pdata->future);
free(pdata);
}
break;
case METHOD_HEAD:
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata);
object_store_head_object(instance_asyn, pdata->future, &getmeta);
break;
case METHOD_DEL:
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_del", del_future_success, del_future_failed, (void *)pdata);
sprintf(pdata->filename, "%s_%u", filecmd->file, index);
object_store_delete_object(instance_asyn, pdata->future, pdata->filename);
break;
case METHOD_PUTONCE:
remain_len = filecont.len;
sprintf(filename_in, "%s_%u", filecmd->file, index);
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_putonce", put_future_success, put_future_failed, (void *)pdata);
if(object_store_upload_once_data(instance_asyn, pdata->future, PUT_MEM_COPY, filecont.buf, filecont.len, &putmeta, pdata->filename, 256))
{
future_destroy(pdata->future);
free(pdata);
}
break;
case METHOD_PUTONCEEV:
remain_len = filecont.len;
sprintf(filename_in, "%s_%u", filecmd->file, index);
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_putonceev", put_future_success, put_future_failed, (void *)pdata);
evbuf = evbuffer_new();
remain_len = filecont.len;
while(remain_len >= 1024)
{
evbuffer_add(evbuf, filecont.buf+(filecont.len-remain_len), 1024);
remain_len -= 1024;
}
if(remain_len > 0)
{
evbuffer_add(evbuf, filecont.buf+(filecont.len-remain_len), remain_len);
}
if(object_store_upload_once_evbuf(instance_asyn, pdata->future, evbuf, &putmeta, pdata->filename, 256))
{
future_destroy(pdata->future);
free(pdata);
}
break;
default:break;
}
index = (index+1) % filecmd->total_num;
}
return NULL;
}
int main(int argc, char **argv)
{
pthread_t thread_tid;
pthread_attr_t attr;
void *runtime_log;
struct filecontentcmd filecmd;
time_t now, remain;
struct event ev_timer;
struct timeval tv;
struct event_base *ev_base;
if(argc != 6)
{
printf("USAGE: %s <method,1-GET,2-PUT,5-PUTONCE> <file> <threads> <total_num> <limit_session_num>\n", argv[0]);
return -1;
}
runtime_log = (void *)MESA_create_runtime_log_handle("./runtime.log", 10);
if(NULL==runtime_log)
{
return -1;
}
filecmd.method = atoi(argv[1]);
filecmd.threads = atoi(argv[3]);
filecmd.total_num = atoi(argv[4]);
filecmd.sess_limit = atoi(argv[5]);
sprintf(filecmd.file, "%s", argv[2]);
object_store_global_init();
instance_asyn = object_store_instance_new("./pangu_tg_cahce.conf", "TANGO_CACHE", filecmd.threads, runtime_log);
assert(instance_asyn!=NULL);
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
for(int i=0; i<4; i++)
{
if(pthread_create(&thread_tid, &attr, thread_transfer_cmd, &filecmd))
{
return -1;
}
}
ev_base = event_base_new();
now = time(NULL);
remain = 10 - (now % 10);
tv.tv_sec = remain;
tv.tv_usec = 0;
evtimer_assign(&ev_timer, ev_base, timer_cb, &ev_timer);
evtimer_add(&ev_timer, &tv);
event_base_dispatch(ev_base);
return 0;
}

333
cache/test/cache_evbase_test.cpp vendored
View File

@@ -1,333 +0,0 @@
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/prctl.h>
#include <sys/un.h>
#include <unistd.h>
#include <sys/poll.h>
#include <curl/curl.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
#include <sys/cdefs.h>
#include <MESA/MESA_handle_logger.h>
#include "cache_evbase_client.h"
struct cache_evbase_instance *instance_asyn;
int runing_over=0;
struct future_pdata
{
struct future * future;
FILE *fp;
char filename[256];
};
void get_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = cache_evbase_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
fwrite(res->data_frag, res->size, 1, pdata->fp);
break;
case RESULT_TYPE_MISS:
printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
if(res->type != RESULT_TYPE_MISS)
printf("get cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
fclose(pdata->fp);
free(pdata);
runing_over = 1;
break;
default:break;
}
}
void get_future_failed(enum e_future_error err, const char * what, void * user)
{
printf("GET fail: %s\n", what);
runing_over = 2;
}
void head_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = cache_evbase_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
assert(0);
break;
case RESULT_TYPE_MISS:
printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
if(res->type != RESULT_TYPE_MISS)
printf("HEAD cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
break;
default:break;
}
}
void head_future_failed(enum e_future_error err, const char * what, void * user)
{
printf("HEAD fail: %s\n", what);
runing_over = 2;
}
void put_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("PUT %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
void put_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("PUT %s fail: %s\n", what, pdata->filename);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
void del_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
void del_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s fail: %s\n", pdata->filename, what);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
char * get_file_content(const char *filename, size_t *filelen_out)
{
char *buffer;
FILE *fp;
size_t filelen = 0;
struct stat filestat;
int readlen;
fp = fopen(filename, "rb");
if(fp == NULL)
{
printf("fopen file %s failed.\n", filename);
return NULL;
}
if(fstat(fileno(fp), &filestat))
{
printf("fstat %s failed.\n", filename);
return NULL;
}
buffer = (char *)malloc(filestat.st_size);
while(filelen < (size_t)filestat.st_size)
{
readlen = fread(buffer + filelen, 1, filestat.st_size - filelen, fp);
if(readlen < 0)
{
printf("read error: %s\n", strerror(errno));
return NULL;
}
filelen += readlen;
}
fclose(fp);
*filelen_out = filestat.st_size;
return buffer;
}
int main(int argc, char **argv)
{
int n, index=0;
char method[16], filename_in[256], filename_out[256], *p;
struct tango_cache_meta_put putmeta;
struct tango_cache_meta_get getmeta;
struct future_pdata *pdata;
struct cache_evbase_ctx *ctx;
void *runtime_log;
struct tango_cache_parameter *parameter;
if(argc != 2 && argc!=3)
{
printf("USGAE: %s <PUT/PUTONCE/PUTONCEEV/GET/DEL:filename> [get_out_file_index]\n", argv[0]);
return -1;
}
if(argc==3)
{
index = atoi(argv[2]);
}
future_promise_library_init(NULL);
runtime_log = (void *)MESA_create_runtime_log_handle("./runtime.log", 10);
if(NULL==runtime_log)
{
return -1;
}
cache_evbase_global_init();
parameter = cache_evbase_parameter_new("./pangu_tg_cahce.conf", "TANGO_CACHE", runtime_log);
assert(parameter != NULL);
instance_asyn = cache_evbase_instance_new(parameter, runtime_log);
assert(instance_asyn!=NULL);
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
if(sscanf(argv[1], "%[^:]:%1023s%n", method, filename_in, &n) != 2)
{
assert(0);
}
if(strlen(filename_in) > 0)
{
memset(&putmeta, 0, sizeof(struct tango_cache_meta_put));
memset(&getmeta, 0, sizeof(struct tango_cache_meta_get));
putmeta.url = filename_in;
putmeta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype";
putmeta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip";
putmeta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
putmeta.usertag_len = strlen(putmeta.usertag);
getmeta.url = filename_in;
p = method;
while(*p=='\r'||*p=='\n') p++;
assert(*p!='\0');
if(!strcasecmp(p, "GET"))
{
sprintf(filename_out, "file_index_%u.bin", index);
pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata);
pdata->fp = fopen(filename_out, "w");
cache_evbase_fetch_object(instance_asyn, pdata->future, &getmeta, OBJECT_IN_UNKNOWN);
}
else if(!strcasecmp(p, "HEAD"))
{
pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata);
cache_evbase_head_object(instance_asyn, pdata->future, &getmeta);
}
else if(!strcasecmp(p, "DEL"))
{
pdata->future = future_create("_del", del_future_success, del_future_failed, (void *)pdata);
sprintf(pdata->filename, "%s", filename_in);
cache_evbase_delete_object(instance_asyn, pdata->future, filename_in);
}
else if(!strcasecmp(p, "PUTONCE"))
{
size_t filelen;
p = get_file_content(filename_in, &filelen);
pdata->future = future_create("_putonce", put_future_success, put_future_failed, pdata);
cache_evbase_upload_once_data(instance_asyn, pdata->future, PUT_MEM_FREE, p, filelen, &putmeta, pdata->filename, 256);
}
else if(!strcasecmp(p, "PUTONCEEV"))
{
size_t readlen;
pdata->future = future_create("_putonceev", put_future_success, put_future_failed, (void *)pdata);
struct evbuffer *evbuf = evbuffer_new();
char buffer[1024];
FILE *fp = fopen(filename_in, "rb");
while(!feof(fp))
{
readlen = fread(buffer, 1, 1024, fp);
if(readlen < 0)
{
assert(0);
}
evbuffer_add(evbuf, buffer, readlen);
}
fclose(fp);
cache_evbase_upload_once_evbuf(instance_asyn, pdata->future, evbuf, &putmeta, pdata->filename, 256);
}
else
{
pdata->future = future_create("_default", put_future_success, put_future_failed, (void *)pdata);
ctx = cache_evbase_update_start(instance_asyn, pdata->future, &putmeta);
char buffer[1024];
FILE *fp = fopen(filename_in, "r");
while(!feof(fp))
{
n = fread(buffer, 1, 1024, fp);
assert(n>=0);
cache_evbase_update_frag_data(ctx, PUT_MEM_COPY, buffer, n);
}
cache_evbase_update_end(ctx, pdata->filename, 256);
}
}
printf("Waiting to finish.......\n");
//static int num=0;
while(!runing_over)
{
/*if(++num==10)
{
cache_evbase_update_end(ctx);
}*/
sleep(1);
}
if(runing_over==2) //GETʱ<54><CAB1>ʱɾ<CAB1><C9BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
{
sleep(5);
}
struct cache_statistics out;
cache_evbase_get_statistics(instance_asyn, &out);
printf("-------------------------------------------------------------------------------------------\n"
"get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n"
"put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n"
"del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num,
out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis,
out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used);
return 0;
}

BIN
cache/test/lib/libcjson.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libcrypto.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libcurl.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libevent.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libhiredis_vip.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libssl.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libtango_cache_client.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libxml2.a vendored
View File

Binary file not shown.

46
cache/test/pangu_tg_cahce.conf vendored
View File

@@ -1,46 +0,0 @@
[TANGO_CACHE]
Addresses of hos, Bucket name in minio. Format is defined by WiredLB.
cache_ip_list=192.168.40.223;
cache_listen_port=9098
cache_bucket_name=hos/proxy_test
#cache_ip_list=192.168.44.10;
#cache_listen_port=9090
#cache_bucket_name=proxybucket
#Maximum number of connections opened by per host.
#max_connection_per_host=1
#Maximum number of requests in a pipeline.
#max_cnnt_pipeline_num=20
#Maximum parellel sessions(http and redis) is allowed to open.
#max_curl_session_num=100
#Maximum time the request is allowed to take(seconds).
#max_curl_transfer_timeout_s=0
#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value.
max_used_memory_size_mb=5120
#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute).
cache_default_ttl_second=3600
#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it.
cache_object_key_hash_switch=1
#Store way: 0-HOS; 1-META in REDIS, object in hos; 2-META and small object in Redis, large object in hos;
cache_store_object_way=0
#If cache_store_object_way is 2 and the size of a object is not bigger than this value, object will be stored in redis.
redis_cache_object_size=20480
#If cache_store_object_way is not 0, we will use redis to store meta and object.
redis_cluster_ip_list=10.4.35.33-34;
redis_cluster_port_range=9001-9016;
#Configs of WiredLB for Minios load balancer.
#wiredlb_override=1
#wiredlb_topic=
#wiredlb_datacenter=
wiredlb_health_port=52102
#wiredlb_group=
log_fsstat_appname=TANGO_CACHE
log_fsstat_filepath=./field_stat.log
log_fsstat_interval=10
log_fsstat_trig=1
log_fsstat_dst_ip=127.0.0.1
log_fsstat_dst_port=8125

445
cache/test/tango_cache_test.cpp vendored
View File

@@ -1,445 +0,0 @@
#include <event2/event.h>
#include <event2/bufferevent.h>
#include <event.h>
#include <event2/event.h>
#include <event2/buffer.h>
#include <event2/http.h>
#include <event2/http_struct.h>
#include <event2/keyvalq_struct.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/prctl.h>
#include <sys/un.h>
#include <unistd.h>
#include <sys/poll.h>
#include <curl/curl.h>
#include <event2/event.h>
#include <event2/event_struct.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
#include <sys/cdefs.h>
#include <MESA/MESA_handle_logger.h>
#include "tango_cache_client.h"
struct event_base *ev_base;
struct tango_cache_instance *tango_instance;
#define MSG_OUT stdout /* Send info to stdout, change to stderr if you want */
struct future_pdata
{
struct future * future;
FILE *fp;
char filename[256];
};
void get_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = tango_cache_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
fwrite(res->data_frag, res->size, 1, pdata->fp);
break;
case RESULT_TYPE_MISS:
printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
if(res->type != RESULT_TYPE_MISS)
printf("get cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
fclose(pdata->fp);
free(pdata);
break;
default:break;
}
}
void get_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
future_destroy(pdata->future);
fclose(pdata->fp);
free(pdata);
printf("GET fail: %s\n", what);
}
void head_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = tango_cache_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
assert(0);
break;
case RESULT_TYPE_MISS:
printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
if(res->type != RESULT_TYPE_MISS)
printf("HEAD cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
free(pdata);
break;
default:break;
}
}
void head_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
future_destroy(pdata->future);
free(pdata);
printf("HEAD fail: %s\n", what);
}
void put_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("PUT %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void put_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("PUT %s fail: %s\n", what, pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void del_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void del_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s fail: %s\n", pdata->filename, what);
future_destroy(pdata->future);
free(pdata);
}
char * get_file_content(const char *filename, size_t *filelen_out)
{
char *buffer;
FILE *fp;
size_t filelen = 0;
struct stat filestat;
int readlen;
fp = fopen(filename, "rb");
if(fp == NULL)
{
printf("fopen file %s failed.\n", filename);
return NULL;
}
if(fstat(fileno(fp), &filestat))
{
printf("fstat %s failed.\n", filename);
return NULL;
}
buffer = (char *)malloc(filestat.st_size);
while(filelen < (size_t)filestat.st_size)
{
readlen = fread(buffer + filelen, 1, filestat.st_size - filelen, fp);
if(readlen < 0)
{
printf("read error: %s\n", strerror(errno));
return NULL;
}
filelen += readlen;
}
fclose(fp);
*filelen_out = filestat.st_size;
return buffer;
}
int tango_cache_multi_delete(struct tango_cache_instance *instance, struct future* future, char *objlist[], u_int32_t num);
static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
{
char s[1024];
long int rv = 0;
int n = 0;
FILE *input = (FILE *)arg;
static int index=0;
char filename[128], method[16], buffer[1024], *p;
char *dellist[16];
char *pstart, *save_ptr=NULL;
int delnum=0;
struct tango_cache_meta_put putmeta;
struct tango_cache_meta_get getmeta;
struct future_pdata *pdata;
struct tango_cache_ctx *ctx;
do
{
s[0]='\0';
rv = fscanf(input, "%[^:]:%1023s%n", method, s, &n);
if(strlen(s) > 0)
{
p = method;
memset(&putmeta, 0, sizeof(struct tango_cache_meta_put));
memset(&getmeta, 0, sizeof(struct tango_cache_meta_get));
putmeta.url = s;
putmeta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype";
putmeta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip";
putmeta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
putmeta.usertag_len = strlen(putmeta.usertag);
getmeta.url = s;
while(*p=='\r'||*p=='\n')p++;
if(*p=='\0') continue;
if(!strcasecmp(p, "GET"))
{
sprintf(filename, "file_index_%u.bin", index++);
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->fp = fopen(filename, "w");
pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata);
if(tango_cache_fetch_object(tango_instance, pdata->future, &getmeta, OBJECT_IN_UNKNOWN) < 0)
{
get_future_failed(FUTURE_ERROR_CANCEL, "", pdata);
}
}
else if(!strcasecmp(p, "HEAD"))
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata);
if(tango_cache_head_object(tango_instance, pdata->future, &getmeta) < 0)
{
head_future_failed(FUTURE_ERROR_CANCEL, "", pdata);
}
}
else if(!strcasecmp(p, "PUTONCE"))
{
size_t filelen;
p = get_file_content(s, &filelen);
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_putnoce", put_future_success, put_future_failed, (void *)pdata);
if(tango_cache_upload_once_data(tango_instance, pdata->future, PUT_MEM_FREE, p, filelen, &putmeta, pdata->filename, 256))
{
put_future_failed(FUTURE_ERROR_CANCEL, "", pdata);
}
}
else if(!strcasecmp(p, "PUTONCEEV"))
{
size_t readlen;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_putonceev", put_future_success, put_future_failed, pdata);
struct evbuffer *evbuf = evbuffer_new();
char buffer[1024];
FILE *fp = fopen(s, "rb");
while(!feof(fp))
{
readlen = fread(buffer, 1, 1024, fp);
if(readlen < 0)
{
assert(0);
}
evbuffer_add(evbuf, buffer, readlen);
}
fclose(fp);
if(tango_cache_upload_once_evbuf(tango_instance, pdata->future, EVBUFFER_MOVE, evbuf, &putmeta, pdata->filename, 256))
{
put_future_failed(FUTURE_ERROR_CANCEL, "", pdata);
}
}
else if(!strcasecmp(p, "DEL"))
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_del", del_future_success, del_future_failed, pdata);
sprintf(pdata->filename, "%s", s);
tango_cache_delete_object(tango_instance, pdata->future, s);
}
else if(!strcasecmp(p, "DELMUL")) //TODO
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_delmul", del_future_success, del_future_failed, pdata);
sprintf(pdata->filename, "%s", s);
for(pstart = strtok_r(s, ";", &save_ptr); pstart != NULL; pstart = strtok_r(NULL, ";", &save_ptr))
{
dellist[delnum++] = pstart;
}
tango_cache_multi_delete(tango_instance, pdata->future, dellist, delnum);
}
else
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create("_default", put_future_success, put_future_failed, pdata);
ctx = tango_cache_update_start(tango_instance, pdata->future, &putmeta);
if(ctx==NULL)
{
put_future_failed(FUTURE_ERROR_CANCEL, "tango_cache_update_start_NULL", pdata);
continue;
}
FILE *fp = fopen(s, "r");
while(!feof(fp))
{
n = fread(buffer, 1, 1024, fp);
assert(n>=0);
tango_cache_update_frag_data(ctx, buffer, n);
}
fclose(fp);
if(tango_cache_update_end(ctx, pdata->filename, 256) < 0)
{
put_future_failed(FUTURE_ERROR_CANCEL, "", pdata);
}
}
}
else
{
break;
}
} while(rv != EOF);
}
struct event fifo_event;
static int init_fifo(void)
{
struct stat st;
curl_socket_t sockfd;
static const char *fifo = "cache.fifo";
fprintf(MSG_OUT, "Creating named pipe \"%s\"\n", fifo);
if(lstat (fifo, &st) == 0) {
if((st.st_mode & S_IFMT) == S_IFREG) {
errno = EEXIST;
perror("lstat");
exit(1);
}
}
unlink(fifo);
if(mkfifo (fifo, 0600) == -1) {
perror("mkfifo");
exit(1);
}
sockfd = open(fifo, O_RDWR | O_NONBLOCK, 0);
if(sockfd == -1) {
perror("open");
exit(1);
}
FILE *input = fdopen(sockfd, "r");
memset(&fifo_event, 0, sizeof(struct event));
fprintf(MSG_OUT, "Now, pipe some URL's into > %s\n", fifo);
event_assign(&fifo_event, ev_base, sockfd, EV_READ|EV_PERSIST,
dummy_accept_callback, input);
event_add(&fifo_event, NULL);
return (0);
}
void timer_cb(evutil_socket_t fd, short what, void *arg)
{
struct timeval tv;
struct cache_statistics out;
tv.tv_sec = 10;
tv.tv_usec = 0;
/*static int num=0;
num++;
if(ctx_global!=NULL && num>5)
{
tango_cache_update_end(ctx_global);
ctx_global = NULL;
}*/
tango_cache_get_statistics(tango_instance, &out);
printf("-------------------------------------------------------------------------------------------\n"
"get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n"
"put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n"
"del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num,
out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis,
out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used);
event_add((struct event *)arg, &tv);
}
int main(int crgc, char **arg)
{
struct event ev_timer;
struct timeval tv;
void *runtime_log;
struct tango_cache_parameter *parameter;
future_promise_library_init(NULL);
runtime_log = (void *)MESA_create_runtime_log_handle("./runtime.log", 10);
if(NULL==runtime_log)
{
return -1;
}
ev_base = event_base_new();
if (!ev_base)
{
printf("create socket error!\n");
return 0;
}
init_fifo();
tango_cache_global_init();
parameter = tango_cache_parameter_new("./pangu_tg_cahce.conf", "TANGO_CACHE", runtime_log);
assert(parameter != NULL);
tango_instance = tango_cache_instance_new(parameter, ev_base, runtime_log);
tv.tv_sec = 10;
tv.tv_usec = 0;
evtimer_assign(&ev_timer, ev_base, timer_cb, &ev_timer);
evtimer_add(&ev_timer, &tv);
event_base_dispatch(ev_base);
return 0;
}

211
cache/test/test_cache_pending.cpp vendored
View File

@@ -1,211 +0,0 @@
#include <iostream>
#include <gtest/gtest.h>
#include <tango_cache_pending.h>
using namespace std;
int main(int argc, char **argv)
{
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(CacheActionTest, PragmaField)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_PRAGMA;
http_heads.value = "no-cache";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1,&restrict), REVALIDATE);
EXPECT_EQ(restrict.min_fresh, 0);
EXPECT_EQ(restrict.max_age, 0);
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), REVALIDATE);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 0);
}
TEST(CacheActionTest, CacheCtlNoCache)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_CACHE_CONTROL;
http_heads.value = "no-cache";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE);
EXPECT_EQ(restrict.min_fresh, 0);
EXPECT_EQ(restrict.max_age, 0);
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), REVALIDATE);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 0);
}
TEST(CacheActionTest, CacheCtlNoStore)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_CACHE_CONTROL;
http_heads.value = "no-store";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), FORBIDDEN);
EXPECT_EQ(restrict.min_fresh, 0);
EXPECT_EQ(restrict.max_age, 0);
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), FORBIDDEN);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 0);
}
TEST(CacheActionTest, CacheCtlCached)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_CACHE_CONTROL;
http_heads.value = "only-if-cached, max-age=3600";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), ALLOWED);
EXPECT_EQ(restrict.min_fresh, 0);
EXPECT_EQ(restrict.max_age, 3600);
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), ALLOWED);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 3600);
}
TEST(CacheActionTest, CacheCtlMinFresh)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
http_heads.http_field = TFE_HTTP_CACHE_CONTROL;
http_heads.value = "only-if-cached, max-age=3600, min-fresh=60";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), ALLOWED);
EXPECT_EQ(restrict.min_fresh, 60);
EXPECT_EQ(restrict.max_age, 3600);
}
TEST(CacheActionTest, CacheCtlMustRevalidate)
{
struct tfe_http_field http_heads;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_CACHE_CONTROL;
http_heads.value = "must-revalidate";
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), REVALIDATE);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 0);
}
TEST(CacheActionTest, CacheCtlProxyRevalidate)
{
struct tfe_http_field http_heads;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_CACHE_CONTROL;
http_heads.value = "proxy-revalidate";
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), REVALIDATE);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 0);
}
TEST(CacheActionTest, CacheCtlCachedSMaxAge)
{
struct tfe_http_field http_heads;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_CACHE_CONTROL;
http_heads.value = "public, max-age=60, s-maxage=3600";
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), ALLOWED);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 3600);
}
TEST(CacheActionTest, CacheCtlPrivate)
{
struct tfe_http_field http_heads;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_CACHE_CONTROL;
http_heads.value = "private";
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), FORBIDDEN);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 0);
}
TEST(CacheActionTest, ExpiresResponse)
{
struct tfe_http_field http_heads;
struct response_freshness freshness;
http_heads.http_field = TFE_HTTP_EXPIRES;
http_heads.value = "Sun, 01 Dec 2019 16:00:00 GMT";
EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), ALLOWED);
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.last_modified, 0);
cout << freshness.timeout << endl;
}
TEST(CacheActionTest, IfMatchRequest)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
http_heads.http_field = TLF_HTTP_IF_MATCH;
http_heads.value = "50b1c1d4f775c61:df3";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE);
EXPECT_EQ(restrict.min_fresh, 0);
EXPECT_EQ(restrict.max_age, 0);
}
TEST(CacheActionTest, IfNoMatchRequest)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
http_heads.http_field = TLF_HTTP_IF_NONE_MATCH;
http_heads.value = "50b1c1d4f775c61:df3";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE);
EXPECT_EQ(restrict.min_fresh, 0);
EXPECT_EQ(restrict.max_age, 0);
}
TEST(CacheActionTest, IfModifiedSinceRequest)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
http_heads.http_field = TLF_HTTP_IF_MODIFIED_SINCE;
http_heads.value = "Sun, 01 Dec 2019 16:00:00 GMT";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE);
EXPECT_EQ(restrict.min_fresh, 0);
EXPECT_EQ(restrict.max_age, 0);
}
TEST(CacheActionTest, IfUnModifiedSinceRequest)
{
struct tfe_http_field http_heads;
struct request_freshness restrict;
http_heads.http_field = TLF_HTTP_IF_UNMODIFIED_SINCE;
http_heads.value = "Sun, 01 Dec 2019 16:00:00 GMT";
EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE);
EXPECT_EQ(restrict.min_fresh, 0);
EXPECT_EQ(restrict.max_age, 0);
}
TEST(CacheActionTest, Date)
{
struct tfe_http_field http_heads[2];
struct response_freshness freshness;
http_heads[0].http_field=TFE_HTTP_CACHE_CONTROL;
http_heads[0].value = "public, max=3600";
http_heads[1].http_field = TFE_HTTP_DATE;
http_heads[1].value = "Sun, 01 Dec 2019 16:00:00 GMT";
EXPECT_EQ(tfe_cache_put_pending(http_heads, 2, &freshness), ALLOWED);
cout << freshness.date << endl;
EXPECT_EQ(freshness.last_modified, 0);
EXPECT_EQ(freshness.timeout, 0);
}
TEST(CacheActionTest, LastModified)
{
struct tfe_http_field http_heads[2];
struct response_freshness freshness;
http_heads[0].http_field=TFE_HTTP_CACHE_CONTROL;
http_heads[0].value = "public, max=3600";
http_heads[1].http_field = TLF_HTTP_LAST_MODIFIED;
http_heads[1].value = "Sun, 01 Dec 2019 16:00:00 GMT";
EXPECT_EQ(tfe_cache_put_pending(http_heads, 2, &freshness),ALLOWED);
cout << freshness.last_modified << endl;
EXPECT_EQ(freshness.date, 0);
EXPECT_EQ(freshness.timeout, 0);
}

3
cache/test/urllist_get.txt vendored
View File

@@ -1,3 +0,0 @@
GET:openbucket/curl_put_chunk_1.pcap
GET:openbucket/tcpdump_all_1.pcap
GET:openbucket/tcpdump_big_1.pcap

3
cache/test/urllist_put.txt vendored
View File

@@ -1,3 +0,0 @@
PUT:openbucket/curl_put_chunk_1.pcap
PUT:openbucket/tcpdump_all_1.pcap
PUT:openbucket/tcpdump_big_1.pcap

3
cmake/Package.cmake
View File

@@ -31,8 +31,7 @@ else()
endif()
# setup %config(noreplace)
set(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/pangu/pangu_pxy.conf"
"%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/tfe.conf"
set(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/tfe.conf"
"%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/zlog.conf"
"%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/future.conf"
"%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/doh/doh.conf")

1
conf/CMakeLists.txt
View File

@@ -1,3 +1,2 @@
install(DIRECTORY tfe DESTINATION conf COMPONENT Profile)
install(DIRECTORY pangu DESTINATION conf COMPONENT Profile)
install(DIRECTORY doh DESTINATION conf COMPONENT Profile)

4
conf/doh/doh.conf
View File

@@ -6,8 +6,8 @@ table_appid=ATTR_APP_ID
table_src_addr=ATTR_SOURCE_ADDR
table_dst_addr=ATTR_DESTINATION_ADDR
table_qname=ATTR_DOH_QNAME
table_host=ATTR_DOH_HOST
table_host_cat=ATTR_DOH_HOST_CAT
table_host=ATTR_SERVER_FQDN
table_host_cat=ATTR_SERVER_FQDN_CAT
[kafka]
ENTRANCE_ID=0

4
conf/tfe/tfe.conf
View File

@@ -116,6 +116,10 @@ cert_expire_time=24
# health_check only for "mode=normal" default 1
enable_health_check=1
[tsg_http]
enable_plugin=1
en_sendlog=1
[debug]
# 1 : enforce tcp passthrough
# 0 : Whether to passthrough depends on the tcp_options in cmsg

2
platform/CMakeLists.txt
View File

@@ -10,7 +10,7 @@ target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external
target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal)
target_include_directories(tfe PRIVATE ${SYSTEMD_INCLUDE_DIRS})
target_link_libraries(tfe common tango-cache-client)
target_link_libraries(tfe common)
target_link_libraries(tfe pthread dl nfnetlink
libnetfilter_queue-static
openssl-ssl-static

3
platform/src/proxy.cpp
View File

@@ -32,7 +32,6 @@
#include <MESA/MESA_prof_load.h>
#include <MESA/field_stat2.h>
#include <tango_cache_client.h>
#include <tfe_utils.h>
#include <tfe_future.h>
#include <tfe_stream.h>
@@ -651,8 +650,6 @@ int main(int argc, char * argv[])
FS_library_init();
future_promise_library_init(future_profile);
tango_cache_global_init();
/* CONFIG */
ret = tfe_proxy_config(g_default_proxy, main_profile);
CHECK_OR_EXIT(ret == 0, "Failed at loading profile %s, Exit.", main_profile);

4
plugin/business/doh/src/doh.cpp
View File

@@ -386,8 +386,8 @@ static int doh_maat_init(const char *profile, const char *section)
MESA_load_profile_string_def(profile, section, "table_src_addr", g_doh_conf->tables[TYPE_SRC_ADDR].name, TFE_STRING_MAX, "ATTR_SOURCE_ADDR");
MESA_load_profile_string_def(profile, section, "table_dst_addr", g_doh_conf->tables[TYPE_DST_ADDR].name, TFE_STRING_MAX, "ATTR_DESTINATION_ADDR");
MESA_load_profile_string_def(profile, section, "table_qname", g_doh_conf->tables[TYPE_QNAME].name, TFE_STRING_MAX, "ATTR_DOH_QNAME");
MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_DOH_HOST");
MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_DOH_HOST_CAT");
MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN");
MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN_CAT");
MESA_load_profile_string_def(profile, section, "table_internal_addr", g_doh_conf->tables[TYPE_INTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_INTERNAL_ADDR");
MESA_load_profile_string_def(profile, section, "table_external_addr", g_doh_conf->tables[TYPE_EXTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_EXTERNAL_ADDR");

6
plugin/business/tsg-http/CMakeLists.txt
View File

@@ -1,7 +1,7 @@
add_library(tsg-http src/tsg_logger.cpp src/tsg_http.cpp src/pattern_replace.cpp src/tsg_web_cache.cpp src/edit_element.cpp src/http_lua.cpp)
add_library(tsg-http src/tsg_logger.cpp src/tsg_http.cpp src/pattern_replace.cpp src/edit_element.cpp src/http_lua.cpp)
target_include_directories(tsg-http PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include)
target_link_libraries(tsg-http PUBLIC common http tango-cache-client)
target_link_libraries(tsg-http PUBLIC rdkafka ctemplate-static cjson pcre2-static ratelimiter-static libdablooms pthread)
target_link_libraries(tsg-http PUBLIC common http)
target_link_libraries(tsg-http PUBLIC rdkafka ctemplate-static cjson pcre2-static libdablooms pthread)
target_link_libraries(tsg-http PUBLIC maatframe tsglua)
target_link_libraries(tsg-http PUBLIC libxml2-static z)

69
plugin/business/tsg-http/include/tsg_proxy_web_cache.h
View File

@@ -1,69 +0,0 @@
#pragma once
#include <event2/event.h>
#include <event2/buffer.h>
#include <tfe_future.h>
#include <MESA/maat.h>
struct cache_handle;
struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section,
struct event_base* gc_evbase, struct maat *feather, void *logger);
struct cached_meta
{
size_t content_length;
char* content_type;
char* last_modified;
char* etag;
};
const struct cached_meta* cache_query_result_read_meta(future_result_t * result);
size_t cache_query_result_get_data(future_result_t * result, const unsigned char** pp_data);
int web_cache_async_read(struct cache_handle* handle, unsigned int thread_id,
const struct tfe_http_half * request, struct cache_mid** mid, struct future* f);
enum cache_query_result_type
{
CACHE_QUERY_RESULT_MISS,
CACHE_QUERY_RESULT_META,
CACHE_QUERY_RESULT_DATA,
CACHE_QUERY_RESULT_END,
CACHE_QUERY_RESULT_IRRELEVANT,
__CACHE_QUERY_RESULT_MAX
};
enum cache_query_result_type cache_query_result_get_type(future_result_t * result);
enum cache_pending_result
{
PENDING_RESULT_NONE=0,
PENDING_RESULT_FOBIDDEN,
PENDING_RESULT_REVALIDATE,
PENDING_RESULT_ALLOWED,
PENDING_RESULT_HIT,
PENDING_RESULT_MISS
};
const char* cache_pending_result_string(enum cache_pending_result);
struct cache_mid;
void cache_mid_clear(struct cache_mid **mid);
const struct cached_meta* cache_pending_result_read_meta(future_result_t * result, struct cache_mid* mid);
enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id,
const struct tfe_http_half * request, struct cache_mid **mid, struct future* f_revalidate);
struct cache_write_context;
struct cache_write_context* web_cache_write_start(struct cache_handle* handle, unsigned int thread_id,
const struct tfe_http_session * session, struct cache_mid **mid);
void web_cache_write(struct cache_write_context* ctx, const unsigned char * body_frag, size_t frag_size);
//return 1 on success
//return -1 on failed
//return -2 on cancel
int web_cache_write_end(struct cache_write_context* ctx);

396
plugin/business/tsg-http/src/tsg_http.cpp
View File

@@ -2,7 +2,6 @@
#include "edit_element.h"
#include "pattern_replace.h"
#include "http_lua.h"
#include "tsg_proxy_web_cache.h"
#include <tfe_proxy.h>
#include <tfe_stream.h>
@@ -31,8 +30,6 @@
#include <string.h>
#include <sys/types.h>
#include "ratelimiter.h"
#define MAX_EDIT_ZONE_NUM 64
enum proxy_action //Bigger action number is prior.
@@ -162,10 +159,6 @@ struct tsg_proxy_rt
int plolicy_table_id[POLICY_PROFILE_TABLE_MAX];
ctemplate::Template * tpl_403, * tpl_404, * tpl_451;
long long suspend_max;
int cache_enabled;
struct cache_handle* cache;
screen_stat_handle_t fs_handle;
long long stat_val[__PX_STAT_MAX];
int fs_id[__PX_STAT_MAX];
@@ -173,7 +166,6 @@ struct tsg_proxy_rt
struct event* gcev;
struct tsg_lua_script lua_script;
Ratelimiter_handle_t ratelimiter;
int enable_rate;
int ctrl_plugin_idx;
@@ -1022,8 +1014,8 @@ int proxy_policy_init(const char* profile_path, const char* static_section, cons
table_name[PXY_CTRL_SOURCE_ADDR] = "ATTR_SOURCE_ADDR";
table_name[PXY_CTRL_DESTINATION_ADDR]="ATTR_DESTINATION_ADDR";
table_name[PXY_CTRL_HTTP_URL] = "ATTR_HTTP_URL";
table_name[PXY_CTRL_HTTP_FQDN] = "ATTR_HTTP_HOST_VIRTUAL";
table_name[PXY_CTRL_HTTP_FQDN_CAT] = "ATTR_HTTP_HOST_CAT_VIRTUAL";
table_name[PXY_CTRL_HTTP_FQDN] = "ATTR_SERVER_FQDN";
table_name[PXY_CTRL_HTTP_FQDN_CAT] = "ATTR_SERVER_FQDN_CAT";
table_name[PXY_CTRL_HTTP_REQ_HDR] = "ATTR_HTTP_REQ_HDR";
table_name[PXY_CTRL_HTTP_REQ_BODY] = "ATTR_HTTP_REQ_BODY";
table_name[PXY_CTRL_HTTP_RES_HDR] = "ATTR_HTTP_RES_HDR";
@@ -1084,64 +1076,12 @@ error_out:
return ret;
}
Ratelimiter_handle_t ratelimit_handle_create(const char* profile_path, const char* static_section)
{
int ret=0, field_stat=0;
int redis_db_idx = 0;
int redis_port = 0, interval_sec = 0;
char redis_server[TFE_STRING_MAX] = {0};
char token_name[TFE_STRING_MAX] = {0};
Ratelimiter_handle_t ratelimit = NULL;
MESA_load_profile_int_def(profile_path, static_section, "enable", &(g_proxy_rt->enable_rate), 0);
MESA_load_profile_int_def(profile_path, static_section, "redis_port", &(redis_port), 6379);
MESA_load_profile_string_def(profile_path, static_section, "redis_server", redis_server, sizeof(redis_server), "");
MESA_load_profile_string_def(profile_path, static_section, "token_name", token_name, sizeof(token_name), "");
MESA_load_profile_int_def(profile_path, static_section, "redis_db_index", &(redis_db_idx), 0);
MESA_load_profile_int_def(profile_path, static_section, "interval_sec", &(interval_sec), 1);
if (g_proxy_rt->enable_rate != 1)
{
return NULL;
}
ratelimit=Ratelimiter_create(token_name, g_proxy_rt->local_logger);
Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_INTERVAL_SEC, &interval_sec, sizeof(interval_sec));
Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_REDIS_IP, redis_server, strlen(redis_server) + 1);
Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_REDIS_PORT, &redis_port, sizeof(redis_port));
Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_REDIS_INDEX, &redis_db_idx, sizeof(redis_db_idx));
/*field stata for debug test**/
MESA_load_profile_int_def(profile_path, static_section, "field_stat", &(field_stat), 0);
if (field_stat == 1)
{
MESA_load_profile_int_def(profile_path, static_section, "stat_port", &(redis_port), 6379);
MESA_load_profile_string_def(profile_path, static_section, "stat_server", redis_server, sizeof(redis_server), "");
Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_FIELD_STAT, &field_stat, sizeof(field_stat));
Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_STAT_IP, redis_server, strlen(redis_server));
Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_STAT_PORT, &redis_port, sizeof(redis_port));
Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_STAT_PATH, token_name, sizeof(token_name));
Ratelimiter_stat_init(ratelimit);
}
ret = Ratelimiter_start(ratelimit);
if (ret < 0)
{
TFE_LOG_ERROR(g_proxy_rt->local_logger, "%s Ratelimiter init failed.", __FUNCTION__);
goto error_out;
}
return ratelimit;
error_out:
Ratelimiter_stop(ratelimit);
return NULL;
}
int proxy_http_init(struct tfe_proxy * proxy)
{
const char * profile_path = "./conf/pangu/pangu_pxy.conf";;
int temp=0;
const char * profile_path = "./conf/tfe/tfe.conf";;
g_proxy_rt = ALLOC(struct tsg_proxy_rt, 1);
MESA_load_profile_int_def(profile_path, "DEBUG", "enable_plugin", &(g_proxy_rt->enable_plugin), 1);
MESA_load_profile_int_def(profile_path, "tsg_http", "enable_plugin", &(g_proxy_rt->enable_plugin), 1);
if (!g_proxy_rt->enable_plugin)
{
return 0;
@@ -1152,7 +1092,7 @@ int proxy_http_init(struct tfe_proxy * proxy)
g_proxy_rt->local_logger = (void *)MESA_create_runtime_log_handle("tsg_http", RLOG_LV_DEBUG);
g_proxy_rt->send_logger = proxy_log_handle_create(profile_path, "LOG", g_proxy_rt->local_logger);
g_proxy_rt->send_logger = proxy_log_handle_create(profile_path, "tsg_http", g_proxy_rt->local_logger);
if (!g_proxy_rt->send_logger)
{
goto error_out;
@@ -1160,7 +1100,6 @@ int proxy_http_init(struct tfe_proxy * proxy)
g_proxy_rt->fs_handle = tfe_proxy_get_fs_handle();
proxy_http_stat_init(g_proxy_rt);
g_proxy_rt->ratelimiter=ratelimit_handle_create(profile_path, "ratelimit");
if(http_lua_handle_create(&g_proxy_rt->lua_script, g_proxy_rt->thread_num, "tfe") <0)
{
goto error_out;
@@ -1183,22 +1122,8 @@ int proxy_http_init(struct tfe_proxy * proxy)
MESA_load_profile_string_def(profile_path, "TEMPLATE", "PAGE_451", page_path, sizeof(page_path),
"./resource/pangu/HTTP451.html");
g_proxy_rt->tpl_451 = ctemplate::Template::GetTemplate(page_path, ctemplate::DO_NOT_STRIP);
g_proxy_rt->enable_rate=0;
MESA_load_profile_int_def(profile_path, "TANGO_CACHE", "suspend_max", &(temp), 1024*1024);
g_proxy_rt->suspend_max=temp;
MESA_load_profile_int_def(profile_path, "TANGO_CACHE", "enable_cache", &(g_proxy_rt->cache_enabled), 1);
if(g_proxy_rt->cache_enabled)
{
g_proxy_rt->cache = create_web_cache_handle(profile_path, "TANGO_CACHE", g_proxy_rt->gc_evbase,
g_proxy_rt->feather, g_proxy_rt->local_logger);
if(!g_proxy_rt->cache)
{
TFE_LOG_INFO(NULL, "Tango Cache init failed.");
goto error_out;
}
TFE_LOG_INFO(NULL, "Tango Cache Enabled.");
}
TFE_LOG_INFO(NULL, "Tsg_pxy HTTP init success.");
return 0;
@@ -1250,7 +1175,6 @@ struct proxy_http_ctx
long long result[MAX_SCAN_RESULT];
struct maat_state *scan_mid;
struct maat_stream *sp;
struct cache_mid* cmid;
struct maat_rule_t * enforce_rules;
size_t n_enforce;
struct policy_action_param *param;
@@ -1267,15 +1191,7 @@ struct proxy_http_ctx
int (* resumed_cb)(const struct tfe_stream * stream,
const struct tfe_http_session * session, enum tfe_http_event event, const unsigned char * data,
size_t datalen, unsigned int thread_id, struct proxy_http_ctx* ctx);
enum cache_pending_result pending_result;
struct future *f_cache_pending, *f_cache_query;
struct tfe_http_session * ref_session;
struct tfe_http_half* cache_revalidate_req;
struct tfe_http_half* cached_response;
size_t cache_result_declared_sz, cache_result_actual_sz;
struct cache_write_context* cache_write_ctx;
int cache_wirte_result;
size_t c2s_byte_num;
size_t s2c_byte_num;
@@ -1398,34 +1314,7 @@ static void proxy_http_ctx_free(struct proxy_http_ctx * ctx)
maat_stream_free(ctx->sp);
ctx->sp=NULL;
}
if(ctx->cache_write_ctx)
{
web_cache_write_end(ctx->cache_write_ctx);
ctx->cache_write_ctx=NULL;
}
//On session recycle
if(ctx->cache_revalidate_req)
{
tfe_http_half_free(ctx->cache_revalidate_req);
ctx->cache_revalidate_req=NULL;
}
if(ctx->cached_response)
{
//Dirty close
ctx->cached_response=NULL;
}
if(ctx->f_cache_query)
{
future_destroy(ctx->f_cache_query);
ctx->f_cache_query=NULL;
}
if(ctx->f_cache_pending)
{
future_destroy(ctx->f_cache_pending);
ctx->f_cache_pending=NULL;
}
if(ctx->log_req_body)
{
evbuffer_free(ctx->log_req_body);
@@ -1472,7 +1361,7 @@ static unsigned long long try_send_by_token(int inject_sz)
{
return 1;
}
return Ratelimiter_customer_factory(g_proxy_rt->ratelimiter, inject_sz);
return inject_sz;
}
static int pangu_action_weight[__PX_ACTION_MAX] = {0};
@@ -2875,248 +2764,6 @@ void enforce_control_policy(const struct tfe_stream * stream, const struct tfe_h
return;
}
#define RESUMED_CB_NO_MORE_CALLS 0
#define RESUMED_CB_MORE_CALLS 1
int make_revalidate_request(const struct tfe_stream * stream, const struct tfe_http_session * session,
enum tfe_http_event events, const unsigned char * body_frag, size_t frag_size, unsigned int thread_id, struct proxy_http_ctx * ctx)
{
assert(ctx->cache_revalidate_req);
if(events & EV_HTTP_REQ_BODY_BEGIN)
{
tfe_http_half_write_body_begin(ctx->cache_revalidate_req, 1);
}
if(events & EV_HTTP_REQ_BODY_CONT)
{
tfe_http_half_write_body_data(ctx->cache_revalidate_req, body_frag, frag_size);
}
if(events & EV_HTTP_REQ_BODY_END)
{
tfe_http_half_write_body_end(ctx->cache_revalidate_req);
ctx->cache_revalidate_req=NULL;
return RESUMED_CB_NO_MORE_CALLS;
}
if(events & EV_HTTP_REQ_END && ctx->cache_revalidate_req)
{
ctx->cache_revalidate_req=NULL;
return RESUMED_CB_NO_MORE_CALLS;
}
return RESUMED_CB_MORE_CALLS;
}
int dummy_resume(const struct tfe_stream * stream, const struct tfe_http_session * session,
enum tfe_http_event events, const unsigned char * body_frag, size_t frag_size, unsigned int thread_id, struct proxy_http_ctx * ctx)
{
return RESUMED_CB_NO_MORE_CALLS;
}
static void cache_read_on_succ(future_result_t * result, void * user)
{
struct proxy_http_ctx * ctx = (struct proxy_http_ctx *)user;
const struct cached_meta* meta=NULL;
enum cache_query_result_type type=cache_query_result_get_type(result);
const unsigned char* data=NULL;
size_t data_sz=0;
char temp[TFE_STRING_MAX];
switch(type)
{
case CACHE_QUERY_RESULT_META:
meta=cache_query_result_read_meta(result);
ctx->cache_result_declared_sz=meta->content_length;
ctx->resumed_cb=dummy_resume;
tfe_http_session_resume(ctx->ref_session);
ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING]));
ctx->cached_response=tfe_http_session_response_create(ctx->ref_session, 200);
tfe_http_std_field_write(ctx->cached_response, TFE_HTTP_CONT_TYPE, meta->content_type);
tfe_http_std_field_write(ctx->cached_response, TFE_HTTP_LAST_MODIFIED, meta->last_modified);
tfe_http_std_field_write(ctx->cached_response, TFE_HTTP_ETAG, meta->etag);
tfe_http_nonstd_field_write(ctx->cached_response, "X-TG-Cache-Lookup", "HIT");
snprintf(temp, sizeof(temp), "%lu", meta->content_length);
tfe_http_std_field_write(ctx->cached_response, TFE_HTTP_CONT_LENGTH, temp);
//Dirty code here.
tfe_http_session_response_set(ctx->ref_session, ctx->cached_response);
//From now, ownership of cached_response has been transfered to http session,
//bussines plugin only hold this pointer as an reference for writing response body.
tfe_http_half_write_body_begin(ctx->cached_response, 1);
meta=NULL;
break;
case CACHE_QUERY_RESULT_DATA:
data_sz=cache_query_result_get_data(result, &data);
tfe_http_half_write_body_data(ctx->cached_response, data, data_sz);
ctx->cache_result_actual_sz+=data_sz;
break;
case CACHE_QUERY_RESULT_END:
assert(ctx->cached_response!=NULL);
tfe_http_half_write_body_end(ctx->cached_response);
//ownership has been transferred to http session, set to NULL.
ctx->pending_result=PENDING_RESULT_HIT;
ctx->cached_response=NULL;
assert(ctx->cache_result_actual_sz==ctx->cache_result_declared_sz);
future_destroy(ctx->f_cache_query);
ctx->f_cache_query=NULL;
break;
case CACHE_QUERY_RESULT_MISS:
ctx->pending_result=PENDING_RESULT_MISS;
ctx->resumed_cb=dummy_resume;
tfe_http_session_resume(ctx->ref_session);
ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING]));
future_destroy(ctx->f_cache_query);
ctx->f_cache_query=NULL;
break;
default:
break;
}
return;
}
static void cache_read_on_fail(enum e_future_error err, const char * what, void * user)
{
struct proxy_http_ctx * ctx = (struct proxy_http_ctx *)user;
future_destroy(ctx->f_cache_query);
ctx->f_cache_query=NULL;
if(!ctx->cached_response)
{
tfe_http_session_resume(ctx->ref_session);
ctx->resumed_cb=dummy_resume;
ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING]));
}
else
{
tfe_http_half_write_body_end(ctx->cached_response);
ctx->cached_response=NULL;
}
ctx->pending_result=PENDING_RESULT_MISS;
printf("cache query failed: %s %s\n", ctx->ref_session->req->req_spec.url, what);
}
static void cache_pend_on_succ(future_result_t * result, void * user)
{
struct proxy_http_ctx * ctx = (struct proxy_http_ctx *)user;
const struct cached_meta* meta=NULL;
meta=cache_pending_result_read_meta(result, ctx->cmid);
ctx->resumed_cb=dummy_resume;
tfe_http_session_resume(ctx->ref_session);
ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING]));
future_destroy(ctx->f_cache_pending);
ctx->f_cache_pending=NULL;
if(meta==NULL)
{
ctx->pending_result = PENDING_RESULT_MISS;
return;
}
if( meta->etag==NULL && meta->last_modified==NULL)
{
ctx->pending_result = PENDING_RESULT_MISS;
return;
}
ctx->pending_result=PENDING_RESULT_REVALIDATE;
struct http_field_name in_field_name;
const char * in_header_value = NULL;
void * iterator = NULL;
ctx->cache_revalidate_req=tfe_http_session_request_create(ctx->ref_session,
ctx->ref_session->req->req_spec.method, ctx->ref_session->req->req_spec.uri);
while (true)
{
if ((in_header_value = tfe_http_field_iterate(ctx->ref_session->req, &iterator, &in_field_name)) == NULL)
{
break;
}
if(in_field_name.field_id==TFE_HTTP_IF_MATCH || in_field_name.field_id==TFE_HTTP_IF_NONE_MATCH
|| in_field_name.field_id==TFE_HTTP_IF_MODIFIED_SINCE
|| in_field_name.field_id==TFE_HTTP_IF_UNMODIFIED_SINCE)
{
continue;
}
tfe_http_field_write(ctx->cache_revalidate_req, &in_field_name, in_header_value);
}
if(meta->etag) tfe_http_std_field_write(ctx->cache_revalidate_req, TFE_HTTP_IF_NONE_MATCH, meta->etag);
if(meta->last_modified) tfe_http_std_field_write(ctx->cache_revalidate_req, TFE_HTTP_IF_MODIFIED_SINCE, meta->last_modified);
tfe_http_session_request_set(ctx->ref_session, ctx->cache_revalidate_req);
ctx->resumed_cb=make_revalidate_request;
return;
}
static void cache_pend_on_fail(enum e_future_error err, const char * what, void * user)
{
struct proxy_http_ctx * ctx = (struct proxy_http_ctx *)user;
ctx->pending_result=PENDING_RESULT_FOBIDDEN;
tfe_http_session_resume(ctx->ref_session);
ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING]));
ctx->resumed_cb=dummy_resume;
future_destroy(ctx->f_cache_pending);
ctx->f_cache_pending=NULL;
return;
}
void cache_pend(const struct tfe_http_session * session, unsigned int thread_id, struct proxy_http_ctx * ctx)
{
if(g_proxy_rt->stat_val[STAT_SUSPENDING]>g_proxy_rt->suspend_max)
{
ctx->pending_result=PENDING_RESULT_FOBIDDEN;
return;
}
ctx->f_cache_pending=future_create("cache_pend", cache_pend_on_succ, cache_pend_on_fail, ctx);
ctx->ref_session=tfe_http_session_allow_write(session);
ctx->pending_result=web_cache_async_pending(g_proxy_rt->cache, thread_id, session->req, &(ctx->cmid), ctx->f_cache_pending);
switch(ctx->pending_result)
{
case PENDING_RESULT_REVALIDATE:
tfe_http_session_suspend(ctx->ref_session);
ATOMIC_INC(&(g_proxy_rt->stat_val[STAT_SUSPENDING]));
break;
case PENDING_RESULT_ALLOWED:
case PENDING_RESULT_FOBIDDEN:
case PENDING_RESULT_MISS:
future_destroy(ctx->f_cache_pending);
ctx->f_cache_pending=NULL;
break;
default:
break;
}
return;
}
void cache_read(const struct tfe_http_session * session, unsigned int thread_id, struct proxy_http_ctx * ctx)
{
if(g_proxy_rt->stat_val[STAT_SUSPENDING]>g_proxy_rt->suspend_max)
{
return;
}
ctx->f_cache_query=future_create("cache_read", cache_read_on_succ, cache_read_on_fail, ctx);
int ret=web_cache_async_read(g_proxy_rt->cache, thread_id, session->req, &(ctx->cmid), ctx->f_cache_query);
if(ret==0)
{
ctx->ref_session=tfe_http_session_allow_write(session);
tfe_http_session_suspend(ctx->ref_session);
ATOMIC_INC(&(g_proxy_rt->stat_val[STAT_SUSPENDING]));
}
else
{
future_destroy(ctx->f_cache_query);
ctx->f_cache_query=NULL;
}
}
void cache_write(const struct tfe_http_session * session, enum tfe_http_event events,
const unsigned char * body_frag, size_t frag_size,
unsigned int thread_id, struct proxy_http_ctx * ctx)
{
if(events & EV_HTTP_RESP_BODY_BEGIN)
{
ctx->cache_write_ctx=web_cache_write_start(g_proxy_rt->cache, thread_id, session, &(ctx->cmid));
}
if(events & EV_HTTP_RESP_BODY_CONT && ctx->cache_write_ctx!=NULL)
{
web_cache_write(ctx->cache_write_ctx, body_frag, frag_size);
}
if(events & EV_HTTP_RESP_BODY_END && ctx->cache_write_ctx!=NULL)
{
ctx->cache_wirte_result=web_cache_write_end(ctx->cache_write_ctx);
ctx->cache_write_ctx=NULL;
}
}
void proxy_on_http_begin(const struct tfe_stream *stream, const struct tfe_http_session *session, unsigned int thread_id, void **pme)
{
@@ -3387,11 +3034,6 @@ void proxy_on_http_end(const struct tfe_stream * stream,
ATOMIC_INC(&(g_proxy_rt->stat_val[STAT_ACTION_RUN_SCRIPT]));
}
TFE_LOG_DEBUG(g_proxy_rt->local_logger, "cache %s %s upload=%d",
session->req->req_spec.url,
cache_pending_result_string(ctx->pending_result),
ctx->cache_wirte_result);
cache_mid_clear(&(ctx->cmid));
proxy_http_ctx_free(ctx);
*pme = NULL;
return;
@@ -3418,30 +3060,6 @@ int proxy_on_http_data(const struct tfe_stream * stream, const struct tfe_http_s
}
enforce_control_policy(stream, session, events, body_frag, frag_size,thread_id, ctx);
if(g_proxy_rt->cache_enabled && ctx->action == PX_ACTION_NONE)
{
if(events & EV_HTTP_REQ_HDR)
{
cache_pend(session, thread_id, ctx);
if(ctx->pending_result==PENDING_RESULT_ALLOWED)
{
cache_read(session, thread_id, ctx);
}
}
if(events & EV_HTTP_RESP_HDR && ctx->pending_result==PENDING_RESULT_REVALIDATE)
{
if(session->resp->resp_spec.resp_code==TFE_HTTP_STATUS_NOT_MODIFIED)
{
cache_read(session, thread_id, ctx);
}
}
if(tfe_http_in_response(events) && ctx->pending_result==PENDING_RESULT_MISS)
{
cache_write(session, events, body_frag, frag_size, thread_id, ctx);
}
}
return NO_CALL_NEXT_PLUGIN;
}

18
plugin/business/tsg-http/src/tsg_logger.cpp
View File

@@ -1,10 +1,12 @@
#include <cjson/cJSON.h>
#include <MESA/MESA_prof_load.h>
#include <tfe_kafka_logger.h>
#include <cache_evbase_client.h>
#include <tfe_utils.h>
#include <tfe_resource.h>
#include <event2/event.h>
#include <event.h>
#include "mpack.h"
#include "tsg_proxy_logger.h"
@@ -16,7 +18,6 @@ struct json_spec
struct proxy_logger
{
int entry_id;
unsigned int en_hoslog;
unsigned int en_sendlog;
const char *device_id;
const char *effective_device_tag;
@@ -113,8 +114,6 @@ struct proxy_logger* proxy_log_handle_create(const char* profile, const char* se
instance->local_logger=local_logger;
TFE_LOG_INFO(local_logger,"Tsg-Pxy log is inititating from %s section %s.", profile, section);
MESA_load_profile_int_def(profile, section, "ENTRANCE_ID",&(instance->entry_id),0);
MESA_load_profile_uint_def(profile, section, "en_hoslog", &instance->en_hoslog, 1);
MESA_load_profile_uint_def(profile, section, "en_sendlog", &instance->en_sendlog, 1);
TFE_LOG_INFO(local_logger, "Tsg-Pxy sendlog : %s", instance->en_sendlog ? "ENABLE" : "DISABLE");
@@ -132,16 +131,6 @@ struct proxy_logger* proxy_log_handle_create(const char* profile, const char* se
goto error_out;
}
if(instance->en_hoslog==1)
{
log_file_upload_para=cache_evbase_parameter_new(profile, section, local_logger);
if (log_file_upload_para == NULL)
{
TFE_LOG_ERROR(local_logger, "Tsg-Pxy failed to new cache evbase parameter.");
goto error_out;
}
instance->log_file_upload_instance=cache_evbase_instance_new(log_file_upload_para, local_logger);
}
return instance;
error_out:
@@ -331,7 +320,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
for(size_t i=0; i<log_msg->result_num; i++)
{
if(log_msg->result[i].do_log!=1) continue;
if(handle->en_hoslog!=1) continue;
if(log_msg->req_body!=NULL)
{

1450
plugin/business/tsg-http/src/tsg_web_cache.cpp
View File

File diff suppressed because it is too large Load Diff

2
resource/pangu/pangu_http.json
View File

@@ -68,7 +68,7 @@
"is_valid": "yes",
"groups": [
{
"virtual_table":"ATTR_HTTP_HOST_VIRTUAL",
"virtual_table":"ATTR_SERVER_FQDN",
"group_name":"http_fqdn",
"group_id":102,
"not_flag":0,

34
resource/pangu/table_info.conf
View File

@@ -129,15 +129,13 @@
},
{
"table_id": 10,
"table_name": "ATTR_HTTP_HOST_VIRTUAL",
"db_tables": ["ATTR_HTTP_HOST", "ATTR_SERVER_FQDN"],
"table_name": "ATTR_SERVER_FQDN",
"table_type": "virtual",
"physical_table": "TSG_OBJ_FQDN"
},
{
"table_id": 11,
"table_name": "ATTR_HTTP_HOST_CAT_VIRTUAL",
"db_tables": ["ATTR_HTTP_HOST_CAT", "ATTR_SERVER_FQDN_CAT"],
"table_name": "ATTR_SERVER_FQDN_CAT",
"table_type": "virtual",
"physical_table": "TSG_OBJ_FQDN_CAT"
},
@@ -457,18 +455,6 @@
},
{
"table_id":43,
"table_name":"ATTR_DOH_HOST",
"table_type":"virtual",
"physical_table": "TSG_OBJ_FQDN"
},
{
"table_id":44,
"table_name":"ATTR_DOH_HOST_CAT",
"table_type":"virtual",
"physical_table": "TSG_OBJ_FQDN_CAT"
},
{
"table_id":45,
"table_name":"PXY_SSL_FINGERPRINT",
"table_type":"plugin",
"valid_column":4,
@@ -478,7 +464,7 @@
}
},
{
"table_id":46,
"table_id":44,
"table_name":"PXY_PROFILE_RUN_SCRIPTS",
"table_type":"plugin",
"valid_column":4,
@@ -489,7 +475,7 @@
}
},
{
"table_id":47,
"table_id":45,
"table_name":"PXY_PROFILE_TCP_OPTION",
"table_type":"plugin",
"valid_column":6,
@@ -499,7 +485,7 @@
}
},
{
"table_id":48,
"table_id":46,
"table_name":"SERVICE_CHAINING_COMPILE",
"table_type":"plugin",
"valid_column":9,
@@ -509,7 +495,7 @@
}
},
{
"table_id": 49,
"table_id": 47,
"table_name": "APP_ID_DICT",
"table_type": "plugin",
"valid_column": 19,
@@ -520,25 +506,25 @@
}
},
{
"table_id":50,
"table_id":48,
"table_name":"ATTR_APP_ID",
"table_type":"virtual",
"physical_table": "APP_ID_DICT"
},
{
"table_id":51,
"table_id":49,
"table_name":"ATTR_SUBSCRIBER_ID",
"table_type":"virtual",
"physical_table": "TSG_OBJ_SUBSCRIBER_ID"
},
{
"table_id":52,
"table_id":50,
"table_name":"ATTR_INTERNAL_ADDR",
"table_type":"virtual",
"physical_table": "TSG_OBJ_IP"
},
{
"table_id":53,
"table_id":51,
"table_name":"ATTR_EXTERNAL_ADDR",
"table_type":"virtual",
"physical_table": "TSG_OBJ_IP"

16
vendor/CMakeLists.txt vendored
View File

@@ -94,22 +94,6 @@ add_dependencies(nghttp2-static nghttp2)
set_property(TARGET nghttp2-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libnghttp2.a)
set_property(TARGET nghttp2-static APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include;${INSTALL_DIR}/src/nghttp2/lib)
### ratelimiter
ExternalProject_Add(ratelimiter PREFIX ratelimiter
URL ${CMAKE_CURRENT_SOURCE_DIR}/ratelimiter-1.1.0-x86_64.tar.gz
URL_MD5 6e4b9b31c70b9e84e2a6434fab6268f9
CONFIGURE_COMMAND ""
BUILD_COMMAND make
BUILD_IN_SOURCE 1)
ExternalProject_Get_Property(ratelimiter INSTALL_DIR)
file(MAKE_DIRECTORY ${INSTALL_DIR}/include)
add_library(ratelimiter-static STATIC IMPORTED GLOBAL)
add_dependencies(ratelimiter-static ratelimiter)
set_property(TARGET ratelimiter-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libratelimiter.a)
set_property(TARGET ratelimiter-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)
#### GoogleTest
ExternalProject_Add(googletest PREFIX googletest
URL ${CMAKE_CURRENT_SOURCE_DIR}/googletest-release-1.8.0.tar.gz

BIN
vendor/ratelimiter-1.1.0-x86_64.tar.gz vendored
View File

Binary file not shown.