gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加集群版redis作为元信息和对象缓存，去除Minio事件通知的redis元信息获取方式。

Browse Source
This commit is contained in:
zhangchengwei
2018-12-14 15:07:09 +08:00
committed by zhengchao
parent 08ae82932a
commit d24c57ce85
32 changed files with 1561 additions and 909 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
cache/test/CMakeLists.txt vendored Normal file
View File

@@ -0,0 +1,22 @@
add_definitions(-fPIC -Wall -g)
add_executable (cache_evbase_test cache_evbase_test.cpp)
target_link_libraries(cache_evbase_test tango_cache_client_static libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (cache_evbase_test MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread z)
add_executable (cache_evbase_benchmark cache_evbase_benchmark.cpp)
target_link_libraries(cache_evbase_benchmark tango_cache_client_static libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (cache_evbase_benchmark MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread z)
#add_executable (cache_evbase_test_threads cache_evbase_test_threads.cpp)
#target_link_libraries(cache_evbase_test_threads tango_cache_client_static libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
#target_link_libraries (cache_evbase_test_threads MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread z)
add_executable (tango_cache_test tango_cache_test.cpp)
target_link_libraries(tango_cache_test tango_cache_client_static libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (tango_cache_test MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread)
#INSTALL (TARGETS cache_evbase_test cache_evbase_test_threads tango_cache_test cache_evbase_benchmark DESTINATION bin)
INSTALL (TARGETS cache_evbase_test tango_cache_test cache_evbase_benchmark DESTINATION bin)
INSTALL (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pangu_tg_cahce.conf DESTINATION bin)
INSTALL (FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmd.txt DESTINATION bin)

12
cache/test/Makefile vendored
View File

@@ -6,17 +6,17 @@ CFLAGS=-Wall -g $(INC_PATH)
LIBS = -lMESA_handle_logger -lMESA_prof_load -lWiredLB
LIBS += -lssl -lcrypto
LIBS += ../lib/libtango_cache_client.a
LIBS += ./lib/libcurl.a ./lib/libevent.a ./lib/libxml2.a ./lib/libhiredis.a ./lib/libhiredis.a
LIBS += ./lib/libcurl.a ./lib/libevent.a ./lib/libxml2.a ./lib/libhiredis_vip.a
OBJS = tango_cache_test.o
OBJS_EVBASE=cache_evbase_test.o
OBJS_EVBASE_THREADS=cache_evbase_test_threads.o
OBJS_EVBASE_BENCHMARK=cache_evbase_benchmark.o
TARGET_EXE=tango_cache_test
TARGET_EXE_EVBASE=cache_evbase_test
TARGET_EXE_EVBASE_THREAD=cache_evbase_test_threads
TARGET_EXE_EVBASE_BENCHMARK=cache_evbase_benchmark
ALL:$(TARGET_EXE) $(TARGET_EXE_EVBASE) $(TARGET_EXE_EVBASE_THREAD)
ALL:$(TARGET_EXE) $(TARGET_EXE_EVBASE) $(TARGET_EXE_EVBASE_BENCHMARK)
$(TARGET_EXE):$(OBJS)
$(CCC) $(LDFLAGS) $^ -o $@ $(LIBS)
@@ -24,7 +24,7 @@ $(TARGET_EXE):$(OBJS)
$(TARGET_EXE_EVBASE):$(OBJS_EVBASE)
$(CCC) $(LDFLAGS) $^ -o $@ $(LIBS) -lpthread
$(TARGET_EXE_EVBASE_THREAD):$(OBJS_EVBASE_THREADS)
$(TARGET_EXE_EVBASE_BENCHMARK):$(OBJS_EVBASE_BENCHMARK)
$(CCC) $(LDFLAGS) $^ -o $@ $(LIBS) -lpthread
.c.o:
@@ -35,5 +35,5 @@ $(TARGET_EXE_EVBASE_THREAD):$(OBJS_EVBASE_THREADS)
-include $(DEPS)
clean:
rm -rf $(OBJS) $(TARGET_EXE) $(OBJS_EVBASE) $(TARGET_EXE_EVBASE) $(OBJS_EVBASE_THREADS) $(TARGET_EXE_EVBASE_THREAD)
rm -rf $(OBJS) $(TARGET_EXE) $(OBJS_EVBASE) $(TARGET_EXE_EVBASE) $(OBJS_EVBASE_BENCHMARK) $(TARGET_EXE_EVBASE_BENCHMARK)

429
cache/test/cache_evbase_benchmark.cpp vendored Normal file
View File

@@ -0,0 +1,429 @@
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/prctl.h>
#include <sys/un.h>
#include <unistd.h>
#include <sys/poll.h>
#include <curl/curl.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
#include <sys/cdefs.h>
#include <MESA/MESA_handle_logger.h>
#include <pthread.h>
#include <sys/prctl.h>
#include "object_store_client.h"
#define METHOD_GET 1
#define METHOD_PUT 2
#define METHOD_HEAD 3
#define METHOD_PUTONCEEV 4
#define METHOD_PUTONCE 5
#define METHOD_DEL 6
struct object_store_instance *instance_asyn;
struct filecontent
{
char *buf;
size_t len;
};
struct future_pdata
{
struct future * future;
FILE *fp;
char filename[256];
};
void get_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = cache_evbase_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
//char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
//memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
//buffer[res->size] = '\0';
//printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
//fwrite(res->data_frag, res->size, 1, pdata->fp);
break;
case RESULT_TYPE_MISS:
//printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
//if(res->type != RESULT_TYPE_MISS)
// printf("get cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
//fclose(pdata->fp);
free(pdata);
break;
default:break;
}
}
void get_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
future_destroy(pdata->future);
free(pdata);
//printf("GET fail: %s\n", what);
}
void head_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = cache_evbase_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
assert(0);
break;
case RESULT_TYPE_MISS:
printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
if(res->type != RESULT_TYPE_MISS)
printf("HEAD cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
free(pdata);
break;
default:break;
}
}
void head_future_failed(enum e_future_error err, const char * what, void * user)
{
printf("HEAD fail: %s\n", what);
}
void put_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
//printf("PUT %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void put_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
//printf("PUT %s fail: %s\n", what, pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void del_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
}
void del_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s fail: %s\n", pdata->filename, what);
future_destroy(pdata->future);
free(pdata);
}
char * get_file_content(const char *filename, size_t *filelen_out)
{
char *buffer;
FILE *fp;
size_t filelen = 0;
struct stat filestat;
int readlen;
fp = fopen(filename, "rb");
if(fp == NULL)
{
printf("fopen file %s failed.\n", filename);
return NULL;
}
if(fstat(fileno(fp), &filestat))
{
printf("fstat %s failed.\n", filename);
return NULL;
}
buffer = (char *)malloc(filestat.st_size);
while(filelen < (size_t)filestat.st_size)
{
readlen = fread(buffer + filelen, 1, filestat.st_size - filelen, fp);
if(readlen < 0)
{
printf("read error: %s\n", strerror(errno));
return NULL;
}
filelen += readlen;
}
fclose(fp);
*filelen_out = filestat.st_size;
return buffer;
}
struct cache_statistics g_out_last;
void timer_cb(evutil_socket_t fd, short what, void *arg)
{
struct timeval tv;
struct cache_statistics out_now;
struct cache_statistics out;
tv.tv_sec = 10;
tv.tv_usec = 0;
/*static int num=0;
num++;
if(ctx_global!=NULL && num>5)
{
tango_cache_update_end(ctx_global);
ctx_global = NULL;
}*/
object_store_get_statistics(instance_asyn, &out_now);
out.del_error_num = out_now.del_error_num - g_out_last.del_error_num;
out.del_recv_num = out_now.del_recv_num - g_out_last.del_recv_num;
out.del_succ_num = out_now.del_succ_num - g_out_last.del_succ_num;
out.get_err_http = out_now.get_err_http - g_out_last.get_err_http;
out.get_err_redis = out_now.get_err_redis - g_out_last.get_err_redis;
out.get_miss_num = out_now.get_miss_num - g_out_last.get_miss_num;
out.get_recv_num = out_now.get_recv_num - g_out_last.get_recv_num;
out.get_succ_http = out_now.get_succ_http - g_out_last.get_succ_http;
out.get_succ_redis = out_now.get_succ_redis - g_out_last.get_succ_redis;
out.put_err_http = out_now.put_err_http - g_out_last.put_err_http;
out.put_err_redis = out_now.put_err_redis - g_out_last.put_err_redis;
out.put_recv_num = out_now.put_recv_num - g_out_last.put_recv_num;
out.put_succ_http = out_now.put_succ_http - g_out_last.put_succ_http;
out.put_succ_redis = out_now.put_succ_redis - g_out_last.put_succ_redis;
out.session_http = out_now.session_http;
out.session_redis = out_now.session_redis;
out.memory_used = out_now.memory_used;
out.totaldrop_num = out_now.totaldrop_num - g_out_last.totaldrop_num;
printf("-------------------------------------------------------------------------------------------\n"
"get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n"
"put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n"
"del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num,
out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis,
out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used);
g_out_last = out_now;
event_add((struct event *)arg, &tv);
}
struct filecontentcmd
{
int method;
int threads;
int total_num;
int sess_limit;
char file[256];
};
static void* thread_transfer_cmd(void *arg)
{
int index=0;
char filename_in[256];
struct tango_cache_meta_put putmeta;
struct tango_cache_meta_get getmeta;
struct future_pdata *pdata;
struct cache_evbase_ctx *ctx;
struct filecontent filecont;
size_t remain_len;
struct cache_statistics out;
struct evbuffer *evbuf;
struct filecontentcmd *filecmd = (struct filecontentcmd *)arg;
prctl(PR_SET_NAME, "transfer_cmd");
memset(&putmeta, 0, sizeof(struct tango_cache_meta_put));
memset(&getmeta, 0, sizeof(struct tango_cache_meta_get));
putmeta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype";
putmeta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip";
putmeta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
putmeta.usertag_len = strlen(putmeta.usertag);
filecont.buf = get_file_content(filecmd->file, &filecont.len);
assert(filecont.buf != NULL);
while(1)
{
object_store_get_statistics(instance_asyn, &out);
if(out.session_http >= filecmd->sess_limit || out.session_redis>=filecmd->sess_limit)
{
usleep(1000);
continue;
}
switch(filecmd->method)
{
case METHOD_GET:
sprintf(filename_in, "%s_%u", filecmd->file, index);
getmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(get_future_success, get_future_failed, pdata);
object_store_fetch_object(instance_asyn, pdata->future, &getmeta, OBJECT_IN_UNKNOWN);
break;
case METHOD_PUT:
remain_len = filecont.len;
sprintf(filename_in, "%s_%u", filecmd->file, index);
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
ctx = object_store_update_start(instance_asyn, pdata->future, &putmeta);
if(ctx == NULL)
{
future_destroy(pdata->future);
free(pdata);
continue;
}
while(remain_len >= 1024)
{
object_store_update_frag_data(ctx, PUT_MEM_COPY, filecont.buf+(filecont.len-remain_len), 1024);
remain_len -= 1024;
}
if(remain_len > 0)
{
object_store_update_frag_data(ctx, PUT_MEM_COPY, filecont.buf+(filecont.len-remain_len), remain_len);
}
object_store_update_end(ctx, pdata->filename, 256);
break;
case METHOD_HEAD:
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(head_future_success, head_future_failed, pdata);
object_store_head_object(instance_asyn, pdata->future, &getmeta);
break;
case METHOD_DEL:
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(del_future_success, del_future_failed, pdata);
sprintf(pdata->filename, "%s_%u", filecmd->file, index);
object_store_delete_object(instance_asyn, pdata->future, pdata->filename);
break;
case METHOD_PUTONCE:
remain_len = filecont.len;
sprintf(filename_in, "%s_%u", filecmd->file, index);
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
object_store_upload_once_data(instance_asyn, pdata->future, PUT_MEM_FREE, filecont.buf, filecont.len, &putmeta, pdata->filename, 256);
break;
case METHOD_PUTONCEEV:
remain_len = filecont.len;
sprintf(filename_in, "%s_%u", filecmd->file, index);
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
evbuf = evbuffer_new();
remain_len = filecont.len;
while(remain_len >= 1024)
{
evbuffer_add(evbuf, filecont.buf+(filecont.len-remain_len), 1024);
remain_len -= 1024;
}
if(remain_len > 0)
{
evbuffer_add(evbuf, filecont.buf+(filecont.len-remain_len), remain_len);
}
object_store_upload_once_evbuf(instance_asyn, pdata->future, evbuf, &putmeta, pdata->filename, 256);
break;
default:break;
}
index = (index+1) % filecmd->total_num;
}
return NULL;
}
int main(int argc, char **argv)
{
pthread_t thread_tid;
pthread_attr_t attr;
void *runtime_log;
struct filecontentcmd filecmd;
struct event ev_timer;
struct timeval tv;
struct event_base *ev_base;
if(argc != 6)
{
printf("USAGE: %s <method,1-GET,2-PUT> <file> <threads> <total_num> <limit_session_num>\n", argv[0]);
return -1;
}
runtime_log = MESA_create_runtime_log_handle("./runtime.log", 10);
if(NULL==runtime_log)
{
return -1;
}
filecmd.method = atoi(argv[1]);
filecmd.threads = atoi(argv[3]);
filecmd.total_num = atoi(argv[4]);
filecmd.sess_limit = atoi(argv[5]);
sprintf(filecmd.file, "%s", argv[2]);
object_store_global_init();
instance_asyn = object_store_instance_new("./pangu_tg_cahce.conf", "TANGO_CACHE", filecmd.threads, runtime_log);
assert(instance_asyn!=NULL);
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
for(int i=0; i<4; i++)
{
if(pthread_create(&thread_tid, &attr, thread_transfer_cmd, &filecmd))
{
return -1;
}
}
ev_base = event_base_new();
tv.tv_sec = 10;
tv.tv_usec = 0;
evtimer_assign(&ev_timer, ev_base, timer_cb, &ev_timer);
evtimer_add(&ev_timer, &tv);
event_base_dispatch(ev_base);
return 0;
}

15
cache/test/cache_evbase_test.cpp vendored
View File

@@ -243,7 +243,7 @@ int main(int argc, char **argv)
pdata->future = future_create(get_future_success, get_future_failed, pdata);
pdata->fp = fopen(filename_out, "w");
cache_evbase_fetch_object(instance_asyn, pdata->future, &getmeta);
cache_evbase_fetch_object(instance_asyn, pdata->future, &getmeta, OBJECT_IN_UNKNOWN);
}
else if(!strcasecmp(p, "HEAD"))
{
@@ -289,8 +289,6 @@ int main(int argc, char **argv)
pdata->future = future_create(put_future_success, put_future_failed, pdata);
ctx = cache_evbase_update_start(instance_asyn, pdata->future, &putmeta);
cache_evbase_get_object_path(ctx, pdata->filename, 256);
char buffer[1024];
FILE *fp = fopen(filename_in, "r");
while(!feof(fp))
@@ -300,7 +298,7 @@ int main(int argc, char **argv)
cache_evbase_update_frag_data(ctx, PUT_MEM_COPY, buffer, n);
}
cache_evbase_update_end(ctx);
cache_evbase_update_end(ctx, pdata->filename, 256);
}
}
@@ -321,8 +319,13 @@ int main(int argc, char **argv)
struct cache_statistics out;
cache_evbase_get_statistics(instance_asyn, &out);
printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_num, out.memory_used);
printf("-------------------------------------------------------------------------------------------\n"
"get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n"
"put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n"
"del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num,
out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis,
out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used);
return 0;
}

322
cache/test/cache_evbase_test_threads.cpp vendored
View File

@@ -1,322 +0,0 @@
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/prctl.h>
#include <sys/un.h>
#include <unistd.h>
#include <sys/poll.h>
#include <curl/curl.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
#include <sys/cdefs.h>
#include <pthread.h>
#include <MESA/MESA_handle_logger.h>
#include "cache_evbase_client.h"
struct cache_evbase_instance *instance_asyn;
int runing_over=0;
struct future_pdata
{
struct future * future;
FILE *fp;
char filename[256];
};
void get_future_success(future_result_t* result, void * user)
{
struct tango_cache_result *res = cache_evbase_read_result(result);
struct future_pdata *pdata = (struct future_pdata *)user;
char buffer[1024];
switch(res->type)
{
case RESULT_TYPE_USERTAG:
case RESULT_TYPE_HEADER:
memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size);
buffer[res->size] = '\0';
printf("%s", buffer);
break;
case RESULT_TYPE_BODY:
fwrite(res->data_frag, res->size, 1, pdata->fp);
break;
case RESULT_TYPE_MISS:
printf("cache not hit/fresh\n");
case RESULT_TYPE_END:
if(res->type != RESULT_TYPE_MISS)
printf("get cache over, total length: %ld\n", res->tlength);
future_destroy(pdata->future);
fclose(pdata->fp);
free(pdata);
runing_over = 1;
break;
default:break;
}
}
void get_future_failed(enum e_future_error err, const char * what, void * user)
{
printf("GET fail: %s\n", what);
runing_over = 2;
}
void put_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("PUT %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
void put_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("PUT %s fail: %s\n", what, pdata->filename);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
void del_future_success(future_result_t* result, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s succ\n", pdata->filename);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
void del_future_failed(enum e_future_error err, const char * what, void * user)
{
struct future_pdata *pdata = (struct future_pdata *)user;
printf("DEL %s fail: %s\n", pdata->filename, what);
future_destroy(pdata->future);
free(pdata);
runing_over = 1;
}
char * get_file_content(const char *filename, size_t *filelen_out)
{
char *buffer;
FILE *fp;
size_t filelen = 0;
struct stat filestat;
int readlen;
fp = fopen(filename, "rb");
if(fp == NULL)
{
printf("fopen file %s failed.\n", filename);
return NULL;
}
if(fstat(fileno(fp), &filestat))
{
printf("fstat %s failed.\n", filename);
return NULL;
}
buffer = (char *)malloc(filestat.st_size);
while(filelen < (size_t)filestat.st_size)
{
readlen = fread(buffer + filelen, 1, filestat.st_size - filelen, fp);
if(readlen < 0)
{
printf("read error: %s\n", strerror(errno));
return NULL;
}
filelen += readlen;
}
fclose(fp);
*filelen_out = filestat.st_size;
return buffer;
}
struct pthread_data
{
char *argv;
int upload_times;
int thread_id;
};
void* thread_upload_download(void *arg)
{
int n;
char method[16], filename_in[256], filename_out[256], *p;
struct tango_cache_meta_put putmeta;
struct tango_cache_meta_get getmeta;
struct future_pdata *pdata;
struct cache_evbase_ctx *ctx;
struct pthread_data *thread_data = (struct pthread_data *)arg;
if(sscanf(thread_data->argv, "%[^:]:%1023s%n", method, filename_in, &n) != 2)
{
assert(0);
}
if(strlen(filename_in) <= 0)
{
return NULL;
}
memset(&putmeta, 0, sizeof(struct tango_cache_meta_put));
putmeta.url = filename_in;
putmeta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype";
putmeta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip";
putmeta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n";
putmeta.usertag_len = strlen(putmeta.usertag);
getmeta.url = filename_in;
p = method;
while(*p=='\r'||*p=='\n') p++;
assert(*p!='\0');
for(int i=0; i<thread_data->upload_times; i++)
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
if(!strcasecmp(p, "GET"))
{
sprintf(filename_out, "file_index_%d_%d.bin", thread_data->thread_id, i);
pdata->future = future_create(get_future_success, get_future_failed, pdata);
pdata->fp = fopen(filename_out, "w");
cache_evbase_fetch_object(instance_asyn, pdata->future, &getmeta);
}
else if(!strcasecmp(p, "DEL"))
{
pdata->future = future_create(del_future_success, del_future_failed, pdata);
sprintf(pdata->filename, "%s", filename_in);
cache_evbase_delete_object(instance_asyn, pdata->future, filename_in);
}
else if(!strcasecmp(p, "PUTONCE"))
{
size_t filelen;
p = get_file_content(filename_in, &filelen);
pdata->future = future_create(put_future_success, put_future_failed, pdata);
if(cache_evbase_upload_once_data(instance_asyn, pdata->future, PUT_MEM_FREE, p, filelen, &putmeta, pdata->filename, 256))
{
printf("cache_evbase_upload_once_data fail: %d\n", cache_evbase_ctx_error(instance_asyn));
future_destroy(pdata->future);
free(pdata);
}
}
else if(!strcasecmp(p, "PUTONCEEV"))
{
size_t readlen;
pdata->future = future_create(put_future_success, put_future_failed, pdata);
struct evbuffer *evbuf = evbuffer_new();
char buffer[1024];
FILE *fp = fopen(filename_in, "rb");
while(!feof(fp))
{
readlen = fread(buffer, 1, 1024, fp);
if(readlen < 0)
{
assert(0);
}
evbuffer_add(evbuf, buffer, readlen);
}
fclose(fp);
if(cache_evbase_upload_once_evbuf(instance_asyn, pdata->future, evbuf, &putmeta, pdata->filename, 256))
{
printf("cache_evbase_upload_once_evbuf fail: %d\n", cache_evbase_ctx_error(instance_asyn));
future_destroy(pdata->future);
free(pdata);
}
evbuffer_free(evbuf);
}
else
{
pdata->future = future_create(put_future_success, put_future_failed, pdata);
ctx = cache_evbase_update_start(instance_asyn, pdata->future, &putmeta);
if(ctx==NULL)
{
printf("cache_evbase_update_start fail: %d\n", cache_evbase_ctx_error(instance_asyn));
future_destroy(pdata->future);
free(pdata);
continue;
}
cache_evbase_get_object_path(ctx, pdata->filename, 256);
char buffer[1024];
FILE *fp = fopen(filename_in, "r");
while(!feof(fp))
{
n = fread(buffer, 1, 1024, fp);
assert(n>=0);
cache_evbase_update_frag_data(ctx, PUT_MEM_COPY, buffer, n);
}
cache_evbase_update_end(ctx);
}
}
printf("transfer over\n");
return NULL;
}
int main(int argc, char **argv)
{
struct cache_statistics out;
void *runtime_log;
pthread_t thread_tid;
pthread_attr_t attr;
struct pthread_data pdata[20];
struct tango_cache_parameter *parameter;
if(argc!=3)
{
printf("USGAE: %s <PUT/PUTONCE/PUTONCEEV/GET/DEL:filename> <how many times>\n", argv[0]);
return -1;
}
runtime_log = MESA_create_runtime_log_handle("./runtime.log", 10);
if(NULL==runtime_log)
{
return -1;
}
cache_evbase_global_init();
parameter = cache_evbase_parameter_new("./pangu_tg_cahce.conf", "TANGO_CACHE", runtime_log);
assert(parameter != NULL);
instance_asyn = cache_evbase_instance_new(parameter, runtime_log);
assert(instance_asyn!=NULL);
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
for(int i=0; i<20; i++)
{
pdata[i].argv = argv[1];
pdata[i].thread_id = i;
pdata[i].upload_times = atoi(argv[2]);
pthread_create(&thread_tid, &attr, thread_upload_download, &pdata[i]);
}
while(1)
{
sleep(30);
cache_evbase_get_statistics(instance_asyn, &out);
printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_num, out.memory_used);
}
return 0;
}

BIN
cache/test/lib/libcjson.a vendored Normal file
View File

Binary file not shown.

BIN
cache/test/lib/libcrypto.a vendored Normal file
View File

Binary file not shown.

BIN
cache/test/lib/libcurl.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libevent.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libhiredis.a vendored
View File

Binary file not shown.

BIN
cache/test/lib/libhiredis_vip.a vendored Normal file
View File

Binary file not shown.

BIN
cache/test/lib/libssl.a vendored Normal file
View File

Binary file not shown.

BIN
cache/test/lib/libxml2.a vendored
View File

Binary file not shown.

57
cache/test/pangu_tg_cahce.conf vendored
View File

@@ -1,40 +1,35 @@
[tango_cache]
#Addresses of minio cluster. Format is defined by WiredLB.
minio_ip_list=192.168.10.61-64;
minio_listen_port=9000
[TANGO_CACHE]
#Addresses of minio. Format is defined by WiredLB.
MINIO_IP_LIST=10.3.35.1;
MINIO_LISTEN_PORT=9000
#Maximum number of connections opened by per host.
#max_connection_per_host=1
#MAX_CONNECTION_PER_HOST=1
#Maximum number of requests in a pipeline.
max_cnnt_pipeline_num=20
#max_curl_session_num=100
#MAX_CNNT_PIPELINE_NUM=20
#Maximum parellel sessions(http and redis) is allowed to open.
#MAX_CURL_SESSION_NUM=100
#Maximum time the request is allowed to take(seconds).
max_curl_transfer_timeout_s=15
#MAX_CURL_TRANSFER_TIMEOUT_S=0
#Bucket name in minio.
cache_bucket_name=openbucket
CACHE_BUCKET_NAME=openbucket
#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value.
max_used_memory_size_mb=5120
MAX_USED_MEMORY_SIZE_MB=5120
#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute).
cache_default_ttl_second=600
#Whether to hash the object key before cache actions. GET/PUT will be faster if you open it.
cache_object_key_hash_switch=1
CACHE_DEFAULT_TTL_SECOND=3600
#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it.
CACHE_OBJECT_KEY_HASH_SWITCH=1
#Where to HEAD meta information of objects. 1-minio;2-redis.
cache_head_from_source=2
#If cache_head_from_source is 2, fill in the following configs.
cache_head_redis_key=MINIO_EVENTS_INFO
#The IP tango_cache_client will always first use to HEAD meta.
cache_head_main_redis_ip=192.168.10.63
#Only when cache_head_main_redis_ip fails will tango_cache_client chose one IP from the list as backup.
cache_head_redis_iplist=192.168.10.62-63;
cache_head_redis_port=6379
#Configs for WiredLB.
#wiredlb_override=1
#wiredlb_topic=
#wiredlb_datacenter=
wiredlb_minio_health_port=52100
#wiredlb_minio_group=
wiredlb_redis_health_port=52101
#wiredlb_redis_group=
#Store way: 0-MINIO; 1-META in REDIS, object in minio; 2-META and small object in Redis, large object in minio;
CACHE_STORE_OBJECT_WAY=2
#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis.
REDIS_CACHE_OBJECT_SIZE=20480
#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
REDIS_CLUSTER_ADDRS=10.4.35.33:9001,10.4.35.34:9001
#Configs of WiredLB for Minios load balancer.
#WIREDLB_OVERRIDE=1
#WIREDLB_TOPIC=
#WIREDLB_DATACENTER=
WIREDLB_HEALTH_PORT=52101
#WIREDLB_GROUP=

15
cache/test/tango_cache_test.cpp vendored
View File

@@ -240,7 +240,7 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
pdata->fp = fopen(filename, "w");
pdata->future = future_create(get_future_success, get_future_failed, pdata);
if(tango_cache_fetch_object(tango_instance, pdata->future, &getmeta) < 0)
if(tango_cache_fetch_object(tango_instance, pdata->future, &getmeta, OBJECT_IN_UNKNOWN) < 0)
{
get_future_failed(FUTURE_ERROR_CANCEL, "", pdata);
}
@@ -315,8 +315,6 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
put_future_failed(FUTURE_ERROR_CANCEL, "tango_cache_update_start_NULL", pdata);
continue;
}
tango_cache_get_object_path(ctx, pdata->filename, 256);
FILE *fp = fopen(s, "r");
while(!feof(fp))
{
@@ -325,7 +323,7 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
tango_cache_update_frag_data(ctx, buffer, n);
}
fclose(fp);
tango_cache_update_end(ctx);
tango_cache_update_end(ctx, pdata->filename, 256);
}
}
else
@@ -387,8 +385,13 @@ void timer_cb(evutil_socket_t fd, short what, void *arg)
}*/
tango_cache_get_statistics(tango_instance, &out);
printf("get_recv: %llu, get_succ: %llu, get_miss: %llu, get_fail: %llu, put_recv: %llu, put_succ: %llu, put_fail: %llu, del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_num, out.get_miss_num, out.get_error_num, out.put_recv_num, out.put_succ_num, out.put_error_num, out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_num, out.memory_used);
printf("-------------------------------------------------------------------------------------------\n"
"get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n"
"put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n"
"del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n",
out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num,
out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis,
out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used);
event_add((struct event *)arg, &tv);
}