gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-5746 使用hos存储非结构化文件

Browse Source 
修复缓存代码编译问题
缓存功能适配hos
This commit is contained in:
fengweihao
2021-03-22 14:10:43 +08:00
parent 62dde9ea6e
commit 2b649f4510
13 changed files with 160 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cache/test/CMakeLists.txt vendored
View File

@@ -1,20 +1,22 @@
add_definitions(-fPIC -Wall -g)
add_executable (cache_evbase_test cache_evbase_test.cpp)
target_link_libraries(cache_evbase_test tango_cache_client_static libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (cache_evbase_test MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread z)
target_include_directories(cache_evbase_test PRIVATE ../include)
target_link_libraries(cache_evbase_test common)
target_link_libraries(cache_evbase_test tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (cache_evbase_test MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread z)
add_executable (cache_evbase_benchmark cache_evbase_benchmark.cpp)
target_link_libraries(cache_evbase_benchmark tango_cache_client_static libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (cache_evbase_benchmark MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread z)
target_link_libraries(cache_evbase_benchmark tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (cache_evbase_benchmark MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread z)
#add_executable (cache_evbase_test_threads cache_evbase_test_threads.cpp)
#target_link_libraries(cache_evbase_test_threads tango_cache_client_static libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
#target_link_libraries(cache_evbase_test_threads tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
#target_link_libraries (cache_evbase_test_threads MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread z)
add_executable (tango_cache_test tango_cache_test.cpp)
target_link_libraries(tango_cache_test tango_cache_client_static libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (tango_cache_test MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread)
target_link_libraries(tango_cache_test tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson)
target_link_libraries (tango_cache_test MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread)
#INSTALL (TARGETS cache_evbase_test cache_evbase_test_threads tango_cache_test cache_evbase_benchmark DESTINATION bin)
INSTALL (TARGETS cache_evbase_test tango_cache_test cache_evbase_benchmark DESTINATION bin)

12
cache/test/cache_evbase_benchmark.cpp vendored
View File

@@ -292,7 +292,7 @@ static void* thread_transfer_cmd(void *arg)
sprintf(filename_in, "%s_%u", filecmd->file, index);
getmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(get_future_success, get_future_failed, pdata);
pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata);
object_store_fetch_object(instance_asyn, pdata->future, &getmeta, OBJECT_IN_UNKNOWN);
break;
@@ -303,7 +303,7 @@ static void* thread_transfer_cmd(void *arg)
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_put", put_future_success, put_future_failed, (void *)pdata);
ctx = object_store_update_start(instance_asyn, pdata->future, &putmeta);
if(ctx == NULL)
{
@@ -329,13 +329,13 @@ static void* thread_transfer_cmd(void *arg)
case METHOD_HEAD:
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(head_future_success, head_future_failed, pdata);
pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata);
object_store_head_object(instance_asyn, pdata->future, &getmeta);
break;
case METHOD_DEL:
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(del_future_success, del_future_failed, pdata);
pdata->future = future_create("_del", del_future_success, del_future_failed, (void *)pdata);
sprintf(pdata->filename, "%s_%u", filecmd->file, index);
object_store_delete_object(instance_asyn, pdata->future, pdata->filename);
break;
@@ -346,7 +346,7 @@ static void* thread_transfer_cmd(void *arg)
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_putonce", put_future_success, put_future_failed, (void *)pdata);
if(object_store_upload_once_data(instance_asyn, pdata->future, PUT_MEM_COPY, filecont.buf, filecont.len, &putmeta, pdata->filename, 256))
{
future_destroy(pdata->future);
@@ -360,7 +360,7 @@ static void* thread_transfer_cmd(void *arg)
putmeta.url = filename_in;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_putonceev", put_future_success, put_future_failed, (void *)pdata);
evbuf = evbuffer_new();
remain_len = filecont.len;

18
cache/test/cache_evbase_test.cpp vendored
View File

@@ -202,7 +202,9 @@ int main(int argc, char **argv)
{
index = atoi(argv[2]);
}
future_promise_library_init(NULL);
runtime_log = MESA_create_runtime_log_handle("./runtime.log", 10);
if(NULL==runtime_log)
{
@@ -240,19 +242,19 @@ int main(int argc, char **argv)
if(!strcasecmp(p, "GET"))
{
sprintf(filename_out, "file_index_%u.bin", index);
pdata->future = future_create(get_future_success, get_future_failed, pdata);
pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata);
pdata->fp = fopen(filename_out, "w");
cache_evbase_fetch_object(instance_asyn, pdata->future, &getmeta, OBJECT_IN_UNKNOWN);
}
else if(!strcasecmp(p, "HEAD"))
{
pdata->future = future_create(head_future_success, head_future_failed, pdata);
pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata);
cache_evbase_head_object(instance_asyn, pdata->future, &getmeta);
}
else if(!strcasecmp(p, "DEL"))
{
pdata->future = future_create(del_future_success, del_future_failed, pdata);
pdata->future = future_create("_del", del_future_success, del_future_failed, (void *)pdata);
sprintf(pdata->filename, "%s", filename_in);
cache_evbase_delete_object(instance_asyn, pdata->future, filename_in);
}
@@ -260,14 +262,14 @@ int main(int argc, char **argv)
{
size_t filelen;
p = get_file_content(filename_in, &filelen);
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_putonce", put_future_success, put_future_failed, pdata);
cache_evbase_upload_once_data(instance_asyn, pdata->future, PUT_MEM_FREE, p, filelen, &putmeta, pdata->filename, 256);
}
else if(!strcasecmp(p, "PUTONCEEV"))
{
size_t readlen;
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_putonceev", put_future_success, put_future_failed, (void *)pdata);
struct evbuffer *evbuf = evbuffer_new();
char buffer[1024];
@@ -286,7 +288,7 @@ int main(int argc, char **argv)
}
else
{
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_default", put_future_success, put_future_failed, (void *)pdata);
ctx = cache_evbase_update_start(instance_asyn, pdata->future, &putmeta);
char buffer[1024];

18
cache/test/pangu_tg_cahce.conf vendored
View File

@@ -1,7 +1,13 @@
[TANGO_CACHE]
#Addresses of minio. Format is defined by WiredLB.
minio_ip_list=10.3.35.60-61;
minio_listen_port=9000
Addresses of hos, Bucket name in minio. Format is defined by WiredLB.
cache_ip_list=192.168.40.223;
cache_listen_port=9098
cache_bucket_name=hos/proxy_test
#cache_ip_list=192.168.44.10;
#cache_listen_port=9090
#cache_bucket_name=proxybucket
#Maximum number of connections opened by per host.
#max_connection_per_host=1
#Maximum number of requests in a pipeline.
@@ -11,8 +17,6 @@ minio_listen_port=9000
#Maximum time the request is allowed to take(seconds).
#max_curl_transfer_timeout_s=0
#Bucket name in minio.
cache_bucket_name=openbucket
#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value.
max_used_memory_size_mb=5120
#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute).
@@ -20,8 +24,8 @@ cache_default_ttl_second=3600
#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it.
cache_object_key_hash_switch=1
#Store way: 0-MINIO; 1-META in REDIS, object in minio; 2-META and small object in Redis, large object in minio;
cache_store_object_way=2
#Store way: 0-HOS; 1-META in REDIS, object in hos; 2-META and small object in Redis, large object in hos;
cache_store_object_way=0
#If cache_store_object_way is 2 and the size of a object is not bigger than this value, object will be stored in redis.
redis_cache_object_size=20480
#If cache_store_object_way is not 0, we will use redis to store meta and object.

22
cache/test/tango_cache_test.cpp vendored
View File

@@ -238,7 +238,7 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
sprintf(filename, "file_index_%u.bin", index++);
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->fp = fopen(filename, "w");
pdata->future = future_create(get_future_success, get_future_failed, pdata);
pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata);
if(tango_cache_fetch_object(tango_instance, pdata->future, &getmeta, OBJECT_IN_UNKNOWN) < 0)
{
@@ -248,7 +248,7 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
else if(!strcasecmp(p, "HEAD"))
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(head_future_success, head_future_failed, pdata);
pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata);
if(tango_cache_head_object(tango_instance, pdata->future, &getmeta) < 0)
{
@@ -260,8 +260,8 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
size_t filelen;
p = get_file_content(s, &filelen);
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_putnoce", put_future_success, put_future_failed, (void *)pdata);
if(tango_cache_upload_once_data(tango_instance, pdata->future, PUT_MEM_FREE, p, filelen, &putmeta, pdata->filename, 256))
{
put_future_failed(FUTURE_ERROR_CANCEL, "", pdata);
@@ -271,7 +271,7 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
{
size_t readlen;
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_putonceev", put_future_success, put_future_failed, pdata);
struct evbuffer *evbuf = evbuffer_new();
char buffer[1024];
@@ -294,14 +294,14 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
else if(!strcasecmp(p, "DEL"))
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(del_future_success, del_future_failed, pdata);
pdata->future = future_create("_del", del_future_success, del_future_failed, pdata);
sprintf(pdata->filename, "%s", s);
tango_cache_delete_object(tango_instance, pdata->future, s);
}
else if(!strcasecmp(p, "DELMUL")) //TODO
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(del_future_success, del_future_failed, pdata);
pdata->future = future_create("_delmul", del_future_success, del_future_failed, pdata);
sprintf(pdata->filename, "%s", s);
for(pstart = strtok_r(s, ";", &save_ptr); pstart != NULL; pstart = strtok_r(NULL, ";", &save_ptr))
@@ -313,8 +313,8 @@ static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg)
else
{
pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata));
pdata->future = future_create(put_future_success, put_future_failed, pdata);
pdata->future = future_create("_default", put_future_success, put_future_failed, pdata);
ctx = tango_cache_update_start(tango_instance, pdata->future, &putmeta);
if(ctx==NULL)
{
@@ -411,7 +411,9 @@ int main(int crgc, char **arg)
struct timeval tv;
void *runtime_log;
struct tango_cache_parameter *parameter;
future_promise_library_init(NULL);
runtime_log = MESA_create_runtime_log_handle("./runtime.log", 10);
if(NULL==runtime_log)
{