gfwleak/pxz-hos-client-cpp-module
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨ feat(src): 重构

Browse Source
This commit is contained in:
彭宣正
2021-05-26 11:10:59 +08:00
parent 678bb1c6f9
commit 8cb339003a
27 changed files with 1068 additions and 965 deletions
Download Patch File Download Diff File Expand all files Collapse all files

478
example/performance/HosClientPerformance.cpp
View File

@@ -15,64 +15,39 @@ extern "C"
#include<sys/stat.h>
#include<math.h>
#include<netinet/in.h>
#include<zlog.h>
}
#include"../../src/hos_client.h"
#include "MESA_handle_logger.h"
#define MAX_THREAD_NUM 32
#ifndef MIN
#define MIN(a,b) ((a) > (b)) ? (b) : (a)
#endif
typedef struct conf_s
{
#define STRING_SIZE 128
char serverip[INET_ADDRSTRLEN];
char bucket[STRING_SIZE];
char object[STRING_SIZE];
char file[STRING_SIZE];
size_t port;
size_t pool_size;
size_t thread_sum;
size_t size;
float append_size;
size_t slice;
int mode;
}conf_t;
#define STRING_SIZE 1024
typedef struct thread_info_s
{
conf_t conf;
hos_client_handle handle;
char object[STRING_SIZE];
char bucket[STRING_SIZE];
char file[100][STRING_SIZE];
int mode;
size_t thread_num;
}thread_info_t;
static void configuration_init(conf_t *conf)
{
strcpy(conf->serverip, "192.168.40.223");
strcpy(conf->bucket, "mybucket");
strcpy(conf->object, "myobject");
strcpy(conf->file, "./file/test.txt");
conf->port = 9098;
conf->pool_size = 4000;
conf->append_size = 1024;
conf->thread_sum = 1;
conf->mode = BUFF_MODE;
conf->slice = 0;
return ;
}
typedef struct userdata_s
{
struct timespec *finished;
}userdata_t;
struct timespec *g_finished;
char g_file_name[100][STRING_SIZE];
size_t g_mode;
size_t g_test_count;
size_t g_append_size;
static size_t calc_time(struct timespec start, struct timespec end)
{
return (end.tv_sec - start.tv_sec) * 1000 * 1000 * 1000 + end.tv_nsec - start.tv_nsec;
}
int read_file_list(const char *path, char file_name[][256])
int read_file_list(const char *path, char file_name[][STRING_SIZE])
{
DIR *dir;
struct dirent *ptr;
@@ -89,12 +64,12 @@ int read_file_list(const char *path, char file_name[][256])
{
if(strcmp(ptr->d_name,".")==0 || strcmp(ptr->d_name,"..")==0) ///current dir OR parrent dir
continue;
else if((ptr->d_type == 8) || (ptr->d_type == 10))
else if((ptr->d_type == DT_REG) || (ptr->d_type == DT_LNK))
{
memcpy(file_name[file_num], path, path_len);
strcat(file_name[file_num], ptr->d_name);
}
else if(ptr->d_type == 4) ///dir
else if(ptr->d_type == DT_DIR) ///dir
{
continue;
}
@@ -135,12 +110,11 @@ static int file_to_buffer(const char *file, char *buffer, size_t *len)
return 0;
}
static int upload_file(char *file, char *buff, int buff_len, int test_times, thread_info_t *thread_info, char *performance_info)
static int upload_file(char *file, char *buff, int buff_len, thread_info_t *thread_info, char *performance_info)
{
int i;
size_t i;
FILE *fp = NULL;
size_t fd[3000];
conf_t *conf = &thread_info->conf;
struct timespec tstart, tend, twrite;
long time_write = 0, time_upload = 0;
size_t len = strlen(performance_info);
@@ -149,11 +123,10 @@ static int upload_file(char *file, char *buff, int buff_len, int test_times, thr
double variance = 0.00;
double average = 0.00;
long time = 0;
int success_cnt;
//写文件
//clock_gettime(CLOCK_MONOTONIC, &tstart);
for (i = 0; i < test_times; i++)
for (i = 0; i < g_test_count; i++)
{
clock_gettime(CLOCK_MONOTONIC, &tstart);
fp = fopen(file, "w+");
@@ -175,14 +148,14 @@ static int upload_file(char *file, char *buff, int buff_len, int test_times, thr
}
//clock_gettime(CLOCK_MONOTONIC, &twrite);
//time_write = calc_time(tstart, twrite);
time_write /= test_times;
time_write /= g_test_count;
//上传文件
//clock_gettime(CLOCK_MONOTONIC, &tstart);
for (i = 0; i < test_times; i++)
for (i = 0; i < g_test_count; i++)
{
clock_gettime(CLOCK_MONOTONIC, &tstart);
fd[i] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
fd[i] = hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, g_mode);
if (hos_write(fd[i], file, 0, thread_info->thread_num) != HOS_CLIENT_OK)
{
printf("error:hos_write file:%s\n", file);
@@ -195,14 +168,14 @@ static int upload_file(char *file, char *buff, int buff_len, int test_times, thr
}
//clock_gettime(CLOCK_MONOTONIC, &tend);
//time_upload = calc_time(tstart, tend);
time_upload /= test_times;
time_upload /= g_test_count;
average = time_write + time_upload;
for (i = 0; i < test_times; i++)
for (i = 0; i < g_test_count; i++)
{
variance += pow((record[i] - average), 2);
}
variance /= test_times;
variance /= g_test_count;
sprintf(file_size, "%dk", buff_len / 1024);
sprintf(&performance_info[len], "%-20lu%-20s%-20ld%-20ld%-20lf%-20lf\n",
@@ -211,143 +184,122 @@ static int upload_file(char *file, char *buff, int buff_len, int test_times, thr
return 0;
}
static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t *thread_info, char *performance_info)
static int upload_buff(char * buff, int buff_len, thread_info_t *thread_info, char *performance_info)
{
int i = 0;
size_t i = 0;
int j = 0;
size_t fd[1000] = {0};
size_t tmp = 0;
size_t rest = 0;
struct timespec tstart, ttmp;
size_t time = 0;
size_t len;
conf_t *conf = &thread_info->conf;
char file_size[128];
char append_size[128];
size_t success_cnt = 0;
int ret = 0;
int drop = 0;
double variance = 0.00;
double average = 0.00;
long record[30000] = {0};
if (conf->slice)
if (g_mode & APPEND_MODE)
{
fd[0] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
for (i = 0; i < test_times; i++)
fd[0] = hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, g_mode);
for (i = 0; i < g_test_count; i++)
{
j = 0;
clock_gettime(CLOCK_MONOTONIC, &tstart);
while(1)
j = 0;
while (1)
{
tmp = j * conf->slice;
tmp = j * g_append_size;
rest = buff_len - tmp;
if (rest < conf->slice)
if (rest < g_append_size)
{
hos_write(fd[0], &buff[tmp], rest, thread_info->thread_num);
break;
}
hos_write(fd[0], &buff[tmp], conf->slice, thread_info->thread_num);
hos_write(fd[0], &buff[tmp], g_append_size, thread_info->thread_num);
j++;
}
clock_gettime(CLOCK_MONOTONIC, &ttmp);
record[i] = calc_time(tstart, ttmp);
average += record[i];
}
average /= test_times;
average /= g_test_count;
for (i = 0; i < test_times; i++)
for (i = 0; i < g_test_count; i++)
{
variance += pow((record[i] - average), 2);
}
variance /= test_times;
variance /= g_test_count;
sprintf(file_size, "%dk", buff_len / 1024);
sprintf(append_size, "%gk", conf->append_size / 1024);
len = strlen(performance_info);
sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20d%-20lf%-20lf\n",
thread_info->thread_num, file_size, append_size, 0, average, sqrt(variance));
}else{
if (conf->mode & APPEND_MODE)
if (buff_len > 1024 * 1024)
{
fd[0] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
#if 1
for (i = 0; i < test_times; i++)
{
clock_gettime(CLOCK_MONOTONIC, &tstart);
#endif
j = 0;
while(1)
{
tmp = j * conf->append_size;
rest = buff_len - tmp;
if (rest < conf->append_size)
{
hos_write(fd[0], &buff[tmp], rest, thread_info->thread_num);
break;
}
hos_write(fd[0], &buff[tmp], conf->append_size, thread_info->thread_num);
j++;
}
#if 1
clock_gettime(CLOCK_MONOTONIC, &ttmp);
record[i] = calc_time(tstart, ttmp);
average += record[i];
}
average /= test_times;
#endif
for (i = 0; i < test_times; i++)
{
variance += pow((record[i] - average), 2);
}
variance /= test_times;
sprintf(file_size, "%dk", buff_len / 1024);
sprintf(append_size, "%gk", conf->append_size / 1024);
len = strlen(performance_info);
sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20d%-20lf%-20lf\n",
thread_info->thread_num, file_size, append_size, 0, average,sqrt(variance));
}else
{
for (i = 0; i < test_times; i++)
{
clock_gettime(CLOCK_MONOTONIC, &tstart);
fd[i] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
ret = hos_write(fd[i], buff, buff_len, thread_info->thread_num);
if (ret == HOS_CLIENT_OK)
{
success_cnt++;
}else
{
//printf("error code:%d, thread_id:%d\n", ret, thread_info->thread_num);
//break;
}
clock_gettime(CLOCK_MONOTONIC, &ttmp);
record[i] = calc_time(tstart, ttmp);
average += record[i];
}
if (success_cnt)
average /= success_cnt;
else
average /= test_times;
for (i = 0; i < test_times; i++)
{
variance += pow((record[i] - average), 2);
}
variance /= test_times;
sprintf(file_size, "%dk", buff_len / 1024);
sprintf(append_size, "%luk", conf->append_size / 1024);
len = strlen(performance_info);
sprintf(&performance_info[len], "%-20lu%-20s%-20d%-20d%-20lf%-20lf\n",
thread_info->thread_num, file_size, 0, 0, average, sqrt(variance));
sprintf(file_size, "%gM", (double)buff_len / 1024 / 1024);
}
else if (buff_len > 1024)
{
sprintf(file_size, "%gK", (double)buff_len / 1024);
}
else
{
sprintf(file_size, "%dB", buff_len);
}
sprintf(append_size, "%gK", (double)g_append_size / 1024);
len = strlen(performance_info);
sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20lu%-20lf%-20lf\n",
thread_info->thread_num, file_size, append_size, g_test_count, average, sqrt(variance));
}
else
{
for (i = 0; i < g_test_count; i++)
{
clock_gettime(CLOCK_MONOTONIC, &tstart);
fd[i] = hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, g_mode);
ret = hos_write(fd[i], buff, buff_len, thread_info->thread_num);
if (ret == HOS_CLIENT_OK)
{
success_cnt++;
}
else
{
//printf("error code:%d, thread_id:%d\n", ret, thread_info->thread_num);
//break;
}
clock_gettime(CLOCK_MONOTONIC, &ttmp);
record[i] = calc_time(tstart, ttmp);
average += record[i];
}
if (success_cnt)
average /= success_cnt;
else
average /= g_test_count;
for (i = 0; i < g_test_count; i++)
{
variance += pow((record[i] - average), 2);
}
variance /= g_test_count;
if (buff_len > 1024 * 1024)
{
sprintf(file_size, "%gM", (double)buff_len / 1024 / 1024);
}
else if (buff_len > 1024)
{
sprintf(file_size, "%gK", (double)buff_len / 1024);
}
else
{
sprintf(file_size, "%dB", buff_len);
}
sprintf(append_size, "%gK", (double)g_append_size / 1024);
len = strlen(performance_info);
sprintf(&performance_info[len], "%-20lu%-20s%-20d%-20lu%-20lf%-20lf\n",
thread_info->thread_num, file_size, 0, g_test_count, average, sqrt(variance));
}
for (i = 0; i < test_times; i++)
for (i = 0; i < g_test_count; i++)
{
if (fd[i] > 2)
{
@@ -362,15 +314,11 @@ static void *put_object_thread(void *ptr)
{
char *performance_info = NULL;
thread_info_t *thread_info = (thread_info_t *)ptr;
conf_t *conf = &thread_info->conf;
char file[128];
size_t buff_len;
int ret;
int i;
char *buff = NULL;
char file_name[100][256];
struct stat s_buf;
int test_times = 1000;
buff = (char *)malloc(30 * 1024 * 1024);
if (buff == NULL)
@@ -387,49 +335,26 @@ static void *put_object_thread(void *ptr)
pthread_exit(NULL);
}
memset(performance_info, 0, 10240);
memset(file_name, 0, 100 *256);
stat(conf->file, &s_buf);
if (S_ISDIR(s_buf.st_mode))
for (i = 0; i < 100; i++)
{
read_file_list(conf->file, file_name);
for (i = 0; i < 100; i++)
{
if (file_name[i][0] == '\0')
break;
ret = file_to_buffer(file_name[i], buff, &buff_len);
if (ret == -1)
{
free(buff);
free(performance_info);
pthread_exit(NULL);
}
if (conf->mode & BUFF_MODE)
{
upload_buff(buff, buff_len, test_times, thread_info, performance_info);
}else
{
sprintf(file, "./file/file_%lu_%d", thread_info->thread_num, i);
upload_file(file, buff, buff_len, test_times, thread_info, performance_info);
}
}
}else
{
ret = file_to_buffer(conf->file, buff, &buff_len);
if (g_file_name[i][0] == '\0')
break;
ret = file_to_buffer(g_file_name[i], buff, &buff_len);
if (ret == -1)
{
free(buff);
free(performance_info);
pthread_exit(NULL);
}
if (conf->mode & BUFF_MODE)
if (g_mode & BUFF_MODE)
{
upload_buff(buff, buff_len, test_times, thread_info, performance_info);
}else
upload_buff(buff, buff_len, thread_info, performance_info);
}
else
{
sprintf(file, "./file/file_%lu", thread_info->thread_num);
upload_file(file, buff, buff_len, test_times, thread_info, performance_info);
sprintf(file, "./file/file_%lu_%d", thread_info->thread_num, i);
upload_file(file, buff, buff_len, thread_info, performance_info);
}
}
free(buff);
@@ -440,92 +365,87 @@ int main(int argc, char *argv[])
{
int ch;
int buf_size;
conf_t conf;
char *object;
char *retval;
size_t thread_num;
size_t thread[MAX_THREAD_NUM];
thread_info_t thread_info[MAX_THREAD_NUM];
cpu_set_t mask;
FILE *log = NULL;
char log_name[256];
const char *log_prefix = "./log/";
char log_name[STRING_SIZE];
const char *log_prefix = "./";
char conf_path[STRING_SIZE] = {0};
char bucket[STRING_SIZE] = {0};
char object[STRING_SIZE] = {0};
char module[STRING_SIZE] = {0};
char upload_file_path[STRING_SIZE] = {0};
size_t thread_sum = 0;
time_t timep;
#if 0
if (argc <= 1)
{
printf("usage: HosClientPerformance \n[-e set endpoint] \n[-b set bucket] \n"
"[-o set object] \n[-f set file] \n[-p set pool size] \n"
"[-t set thread sum] \n[-B set BUFF_MODE] \n"
"[-F set FILE_MODE] \n[-A set APPEND_MODE] \n"
"[-h show help info] \n");
return -1;
}
#endif
configuration_init(&conf);
struct stat s_buf;
/*init*/
g_append_size = 102400;
memcpy(bucket, "hos_test_bucket", strlen("hos_test_bucket"));
memcpy(object, "object", strlen("object"));
memcpy(conf_path, "../conf/default.conf", strlen("../conf/default.conf"));
memcpy(module, "module", strlen("module"));
memcpy(upload_file_path, "../CMakeLists.txt", strlen("../CMakeLists.txt"));
thread_sum = 1;
g_mode = BUFF_MODE;
g_test_count = 100;
//读取命令行配置
while((ch = getopt(argc, argv, "a:b:o:f:p:t:k:s:i:P:S:BFAh")) != -1)
while((ch = getopt(argc, argv, "a:b:c:o:m:f:t:M:n:h")) != -1)
{
switch(ch)
{
case 'a':
conf.append_size = 1024 * atof(optarg);
g_append_size = atoi(optarg);
break;
case 'i':
//endpoint
buf_size = MIN(STRING_SIZE, strlen(optarg));
strncpy((char *)conf.serverip, optarg, buf_size);
break;
case 'b':
buf_size = MIN(STRING_SIZE, strlen(optarg));
strncpy((char *)conf.bucket, optarg, buf_size);
conf.bucket[buf_size] = '\0';
break;
case 'o':
case 'b': /*bucket*/
buf_size = MIN(STRING_SIZE, strlen(optarg));
strncpy(conf.object, optarg, buf_size);
conf.object[buf_size] = '\0';
strncpy(bucket, optarg, buf_size);
bucket[buf_size] = '\0';
break;
case 'f':
case 'c': /*configuration file*/
buf_size = MIN(STRING_SIZE, strlen(optarg));
strncpy(conf.file, optarg, buf_size);
conf.file[buf_size] = '\0';
strncpy(conf_path, optarg, buf_size);
conf_path[buf_size] = '\0';
break;
case 'P':
conf.pool_size = atoi(optarg);
conf.pool_size = MIN(4000, conf.pool_size);
case 'o': /*object*/
buf_size = MIN(STRING_SIZE, strlen(optarg));
strncpy(object, optarg, buf_size);
object[buf_size] = '\0';
break;
case 'm': /*module*/
buf_size = MIN(STRING_SIZE, strlen(optarg));
strncpy(module, optarg, buf_size);
module[buf_size] = '\0';
break;
case 'f': /*module*/
buf_size = MIN(STRING_SIZE, strlen(optarg));
strncpy(upload_file_path, optarg, buf_size);
upload_file_path[buf_size] = '\0';
break;
case 't':
conf.thread_sum = atoi(optarg);
thread_sum = atoi(optarg);
break;
case 'B':
conf.mode &= BUFF_MODE;
case 'M':
g_mode = atoi(optarg);
break;
case 'F':
conf.mode &= FILE_MODE;
break;
case 'A':
conf.mode |= APPEND_MODE;
break;
case 'k':
conf.append_size = 1024 * atoi(optarg);
break;
case 's':
conf.size = atoi(optarg);
break;
case 'S':
conf.slice = atoi(optarg);
break;
case 'p':
conf.port = atoi(optarg);
case 'n':
g_test_count = atoi(optarg);
break;
case 'h':
default:
printf("usage: HosClientPerformance \n[-e set endpoint] \n[-b set bucket] \n"
"[-o set object] \n[-f set file] \n[-p set pool size] \n"
"[-t set thread sum] \n[-B set BUFF_MODE] \n"
"[-F set FILE_MODE] \n[-A set APPEND_MODE] \n"
"[-h show help info] \n[-a set append size(K)]\n");
printf("usage: HosClientPerformance \n"
"[-b set bucket] \n"
"[-c set conf file path] \n"
"[-o set object] \n"
"[-t set thread sum] \n"
"[-m set module] \n"
"[-f set upload file path] \n"
"[-M set mode] \n"
"[-h show help info] \n");
return -1;
break;
}
@@ -540,50 +460,52 @@ int main(int argc, char *argv[])
perror(log_name);
return -1;
}
//创建client
hos_client_handle handle = hos_client_create(conf.serverip, conf.port, "default", "default", conf.pool_size);
if (handle == NULL)
//初始化hos instance
hos_instance hos_instance = hos_init_instance(conf_path, module, thread_sum, bucket);
if (hos_instance == NULL)
{
printf("error:hos_client_handle\n");
printf("error:hos_client_handle\n %s\n", hos_instance->error_message);
fclose(log);
return -1;
}
hos_set_thread_sum(handle, conf.thread_sum);
hos_set_cache_size(handle, conf.append_size);
hos_set_cache_count(handle, 0);
//创建bucket
if (hos_create_bucket(handle, conf.bucket))
{
printf("error:hos_create_bucket\n");
fclose(log);
hos_client_destory(handle);
return -1;
}
//zlog_init("../conf/zlog.conf");
MESA_handle_runtime_log_creation(NULL);
printf("\n==============================================================================================================================\n");
if (conf.mode & BUFF_MODE)
if (g_mode & BUFF_MODE)
{
printf("%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "append_size", "upload_time", "total_time");
printf("%-20s%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "append_size", "upload_time", "total_time", "std-dev");
}else
{
printf("%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "write_time", "upload_time", "total_time");
printf("%-20s%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "write_time", "upload_time", "total_time", "std-dev");
}
hos_expand_fs2(handle, "./log/fs2.log", 0, "127.0.0.1", 8001);
//hos_expand_fs2(handle, NULL, 0, "127.0.0.1", 8001);
for ( thread_num = 0; thread_num < conf.thread_sum; thread_num++ )
memset(g_file_name, 0, 100 * 256);
stat(upload_file_path, &s_buf);
if (S_ISDIR(s_buf.st_mode))
{
read_file_list(upload_file_path, g_file_name);
for (int i = 0; i < 100; i++)
{
if (g_file_name[i][0] == '\0')
break;
}
}else
{
memcpy(g_file_name[0], upload_file_path, MIN(strlen(upload_file_path), STRING_SIZE - 1));
}
for ( thread_num = 0; thread_num < thread_sum; thread_num++ )
{
thread_info[thread_num].conf = conf;
object = thread_info[thread_num].conf.object;
sprintf(&object[strlen(object)], "-%lu", thread_num);
thread_info[thread_num].thread_num = thread_num;
thread_info[thread_num].handle = handle;
sprintf(thread_info[thread_num].object, "%s-%lu", object, thread_num);
sprintf(thread_info[thread_num].bucket, "%s", bucket);
if(pthread_create(&thread[thread_num], NULL, put_object_thread, (void *)&thread_info[thread_num]))
{
perror(" ");
fclose(log);
hos_client_destory(handle);
hos_shutdown_instance();
return -1;
}
@@ -595,7 +517,7 @@ int main(int argc, char *argv[])
}
}
for (thread_num = 0; thread_num < conf.thread_sum; thread_num++)
for (thread_num = 0; thread_num < thread_sum; thread_num++)
{
pthread_join(thread[thread_num], (void **)&retval);
if (retval)
@@ -606,7 +528,7 @@ int main(int argc, char *argv[])
}
}
if (hos_client_destory(handle) == 0)
if (hos_shutdown_instance() == 0)
{
//time = calc_time(start, finished);
//time /= test_times;