gfwleak/pxz-hos-client-cpp-module
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🐞 fix(TSG-9807): 修复cache_size设置为0导致的内存快速消耗

Browse Source
This commit is contained in:
“pengxuanzheng”
2022-03-02 10:35:26 +00:00
parent d4e8b149c8
commit 16d71d2fe6
22 changed files with 214 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
example/demo/CMakeLists.txt
View File

@@ -5,7 +5,10 @@ project(hos_write_demo)
SET(CMAKE_BUILD_TYPE Debug)
link_directories(/opt/MESA/lib/)
include_directories(/opt/MESA/include)
include_directories("${CMAKE_SOURCE_DIR}/src/")
link_directories("${CMAKE_BINARY_DIR}/src")
add_executable(hos_write_demo hos_write_demo.cpp)
add_dependencies(hos_write_demo ${lib_name}_shared)
target_link_libraries(hos_write_demo hos-client-cpp)

2
example/demo/conf/default.conf
View File

@@ -1,5 +1,5 @@
[hos_default_conf]
hos_serverip=192.168.10.1
hos_serverip=192.168.44.67
hos_serverport=9098
hos_accesskeyid="default"
hos_secretkey="default"

72
example/demo/hos_write_demo.cpp
View File

@@ -18,19 +18,18 @@ extern "C"
typedef struct userdata_s
{
struct timespec *finished;
//struct timespec *finished;
}userdata_t;
static size_t calc_time(struct timespec start, struct timespec end)
{
return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec -
(start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
}
// static size_t calc_time(struct timespec start, struct timespec end)
// {
// return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec -
// (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
// }
int file_to_buffer(const char *file, char *buffer, int size)
{
FILE *fp = fopen(file, "r");
int num = 0;
if (fp == NULL)
{
@@ -38,7 +37,7 @@ int file_to_buffer(const char *file, char *buffer, int size)
return -1;
}
do{
num = fread(buffer, 1, size, fp);
int num = fread(buffer, 1, size, fp);
if (num < 0)
{
return -1;
@@ -56,30 +55,24 @@ void callback(bool result, const char *bucket, const char *object, const char *e
{
if (!result)
{
printf("error:[%d], %s\n", errorcode, errormsg);
printf("error:[%zu], %s\n", errorcode, errormsg);
}
return ;
}
int main(int argc, char *argv[])
int main(const int argc, const char *argv[])
{
if (argc != 4)
{
printf("usege: [conf file] [module name] [file name]\n");
return -1;
}
struct timespec start, end, finished;
size_t time;
int i = 0;
char *conf_file = argv[1];
char *module_name = argv[2];
char *file_name = argv[3];
const char *conf_file = argv[1];
const char *module_name = argv[2];
const char *file_name = argv[3];
struct stat buffer;
char *buf = NULL;
size_t buf_size;
int mode = FILE_MODE;
size_t fd = 0;
userdata_t data = {&finished};
hos_instance hos_instance = NULL;
char object[1024];
const char *bucket = "hos_test_bucket";
@@ -107,7 +100,7 @@ int main(int argc, char *argv[])
if (hos_instance == NULL)
{
printf("error:hos_init_instance\n");
printf("error:[%d]%s", hos_get_init_instance_errorcode, hos_get_init_instance_errormsg);
printf("error:[%d]%s", hos_get_init_instance_errorcode(), hos_get_init_instance_errormsg());
return -1;
}
printf("hos_init_instance success ... \n");
@@ -128,19 +121,48 @@ int main(int argc, char *argv[])
}
printf("hos_upload_buff end ...\n");
mode = BUFF_MODE | APPEND_MODE;
printf("hos_write buff start ...\n");
snprintf(object, 1023, "%s_write_APPEND", file_name);
int err = hos_open_fd(bucket, object, callback, NULL, 0, &fd);
if (hos_write(fd, buf, buffer.st_size) != HOS_CLIENT_OK)
hos_open_fd(bucket, object, callback, NULL, 0, &fd);
int ret = hos_write(fd, buf, buffer.st_size);
if (ret == HOS_CLIENT_OK)
{
printf("debug: hos_write successed 1st.\n");
}
else if (ret == HOS_IN_CACHE)
{
printf("debug: hos_write in cache.\n");
}
else
{
printf("error: hos_write failed 1st!\n");
}
if (hos_write(fd, buf, buffer.st_size) != HOS_CLIENT_OK)
ret = hos_write(fd, buf, buffer.st_size);
if (ret == HOS_CLIENT_OK)
{
printf("debug: hos_write successed 2nd.\n");
}
else if (ret == HOS_IN_CACHE)
{
printf("debug: hos_write in cache.\n");
}
else
{
printf("error: hos_write failed 2nd!\n");
}
hos_close_fd(fd);
ret = hos_close_fd(fd);
if (ret == HOS_CLIENT_OK)
{
printf("debug: hos_close_fd successed.\n");
}
else if (ret == HOS_FD_CLOSE_BUT_SEND_FAILED)
{
printf("debug: hos send fialed, but close fd successed.\n");
}
else
{
printf("error: hos_close_fd failed.\n");
}
printf("hos_write buff end ...\n");
printf("hos_shutdown_instance start ...\n");

4
example/performance/CMakeLists.txt
View File

@@ -7,6 +7,9 @@ link_directories(/usr/local/lib64/)
link_directories(/opt/MESA/lib/)
link_libraries(hos-client-cpp pthread)
include_directories(/opt/MESA/include)
include_directories(/opt/MESA/include/MESA)
include_directories("${CMAKE_SOURCE_DIR}/src/")
link_directories("${CMAKE_BINARY_DIR}/src")
#for ASAN
set(ASAN_OPTION "OFF" CACHE STRING " set asan type chosen by the user, using OFF as default")
@@ -27,5 +30,6 @@ endif()
# end of for ASAN
add_executable(HosClientPerformance HosClientPerformance.cpp)
add_dependencies(HosClientPerformance ${lib_name}_shared)
target_link_libraries(HosClientPerformance hos-client-cpp MESA_handle_logger)

36
example/performance/HosClientPerformance.cpp
View File

@@ -121,8 +121,6 @@ static int upload_file(char *file, char *buff, int buff_len, thread_info_t *thre
long record[1000] = {0};
double variance = 0.00;
double average = 0.00;
long time = 0;
int err = 0;
//写文件
//clock_gettime(CLOCK_MONOTONIC, &tstart);
@@ -155,14 +153,14 @@ static int upload_file(char *file, char *buff, int buff_len, thread_info_t *thre
for (i = 0; i < g_test_count; i++)
{
clock_gettime(CLOCK_MONOTONIC, &tstart);
err = hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, &fd[i]);
hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, &fd[i]);
if (hos_write(fd[i], file, 0) != HOS_CLIENT_OK)
{
printf("error:hos_write file:%s\n", file);
return -1;
}
clock_gettime(CLOCK_MONOTONIC, &tend);
time = calc_time(tstart, tend);
long time = calc_time(tstart, tend);
time_upload += time;
record[i] += time;
}
@@ -189,7 +187,7 @@ static int upload_file(char *file, char *buff, int buff_len, thread_info_t *thre
{
sprintf(file_size, "%dB", buff_len);
}
sprintf(&performance_info[len], "%-20lu%-20s%-20ld%-20ld%-20lf%-20lf\n",
sprintf(&performance_info[len], "%-20zu%-20s%-20ld%-20ld%-20lf%-20lf\n",
thread_info->thread_num, file_size, time_write, time_upload, average, sqrt(variance));
return 0;
@@ -198,33 +196,27 @@ static int upload_file(char *file, char *buff, int buff_len, thread_info_t *thre
static int upload_buff(char * buff, int buff_len, thread_info_t *thread_info, char *performance_info)
{
size_t i = 0;
int j = 0;
size_t fd[1000] = {0};
size_t tmp = 0;
size_t rest = 0;
struct timespec tstart, ttmp;
size_t len;
char file_size[128];
char append_size[128];
size_t success_cnt = 0;
int ret = 0;
double variance = 0.00;
double average = 0.00;
long record[30000] = {0};
int err = 0;
if (g_mode & APPEND_MODE)
{
err = hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, &fd[0]);
hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, &fd[0]);
for (i = 0; i < g_test_count; i++)
{
clock_gettime(CLOCK_MONOTONIC, &tstart);
j = 0;
int j = 0;
while (1)
{
tmp = j * g_append_size;
rest = buff_len - tmp;
size_t tmp = j * g_append_size;
size_t rest = buff_len - tmp;
if (rest <= g_append_size)
{
hos_write(fd[0], &buff[tmp], rest);
@@ -259,15 +251,16 @@ static int upload_buff(char * buff, int buff_len, thread_info_t *thread_info, ch
}
sprintf(append_size, "%gK", (double)g_append_size / 1024);
len = strlen(performance_info);
sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20lu%-20lf%-20lf\n",
sprintf(&performance_info[len], "%-20zu%-20s%-20s%-20zu%-20lf%-20lf\n",
thread_info->thread_num, file_size, append_size, g_test_count, average, sqrt(variance));
}
else
{
size_t success_cnt = 0;
for (i = 0; i < g_test_count; i++)
{
clock_gettime(CLOCK_MONOTONIC, &tstart);
ret = hos_upload_buf(thread_info->bucket, thread_info->object, buff, buff_len, callback, NULL, thread_info->thread_num);
int ret = hos_upload_buf(thread_info->bucket, thread_info->object, buff, buff_len, callback, NULL, thread_info->thread_num);
if (ret == HOS_CLIENT_OK)
{
success_cnt++;
@@ -306,7 +299,7 @@ static int upload_buff(char * buff, int buff_len, thread_info_t *thread_info, ch
}
sprintf(append_size, "%gK", (double)g_append_size / 1024);
len = strlen(performance_info);
sprintf(&performance_info[len], "%-20lu%-20s%-20d%-20lu%-20lf%-20lf\n",
sprintf(&performance_info[len], "%-20zu%-20s%-20d%-20zu%-20lf%-20lf\n",
thread_info->thread_num, file_size, 0, g_test_count, average, sqrt(variance));
}
@@ -327,7 +320,6 @@ static void *put_object_thread(void *ptr)
thread_info_t *thread_info = (thread_info_t *)ptr;
char file[128];
size_t buff_len;
int ret;
int i;
char *buff = NULL;
@@ -351,7 +343,7 @@ static void *put_object_thread(void *ptr)
{
if (g_file_name[i][0] == '\0')
break;
ret = file_to_buffer(g_file_name[i], buff, &buff_len);
int ret = file_to_buffer(g_file_name[i], buff, &buff_len);
if (ret == -1)
{
free(buff);
@@ -364,7 +356,7 @@ static void *put_object_thread(void *ptr)
}
else
{
sprintf(file, "./file/file_%lu_%d", thread_info->thread_num, i);
sprintf(file, "./file/file_%zu_%d", thread_info->thread_num, i);
upload_file(file, buff, buff_len, thread_info, performance_info);
}
}
@@ -509,7 +501,7 @@ int main(int argc, char *argv[])
for ( thread_num = 0; thread_num < thread_sum; thread_num++ )
{
thread_info[thread_num].thread_num = thread_num;
sprintf(thread_info[thread_num].object, "%s-%lu", object, thread_num);
sprintf(thread_info[thread_num].object, "%s-%zu", object, thread_num);
sprintf(thread_info[thread_num].bucket, "%s", bucket);
if(pthread_create(&thread[thread_num], NULL, put_object_thread, (void *)&thread_info[thread_num]))