diff --git a/example/data/test_size.sh b/example/data/test_size.sh deleted file mode 100755 index 8fbb5533..00000000 --- a/example/data/test_size.sh +++ /dev/null @@ -1,18 +0,0 @@ -######################################################################### -# File Name: test_times.sh -# Author: pxz -# Created Time: Mon 21 Sep 2020 04:43:35 PM CST -######################################################################### -#!/bin/bash -test_size=("1k" "10k" "100k" "1M" "2M" "3M" "4M" "5M" "10M" "20M" "30M") -num=0 - -while((${num} < 11)) -do - #echo ./hos_write_complete mybucket ${test_size[$num]}.data 10000 - #./hos_write_complete mybucket ${test_size[$num]}.data 10000 - echo ./hos_upload_complete mybucket ${test_size[$num]}.data 10000 - ./hos_upload_complete mybucket ${test_size[$num]}.data 10000 - let "num++" -done - diff --git a/example/CMakeLists.txt b/example/demo/CMakeLists.txt similarity index 100% rename from example/CMakeLists.txt rename to example/demo/CMakeLists.txt diff --git a/example/data/100k.data b/example/demo/data/100k.data similarity index 100% rename from example/data/100k.data rename to example/demo/data/100k.data diff --git a/example/demo/data/10M.data b/example/demo/data/10M.data new file mode 100644 index 00000000..6c5d4031 Binary files /dev/null and b/example/demo/data/10M.data differ diff --git a/example/data/10k.data b/example/demo/data/10k.data similarity index 100% rename from example/data/10k.data rename to example/demo/data/10k.data diff --git a/example/demo/data/11k.data b/example/demo/data/11k.data new file mode 100644 index 00000000..258e2e6c Binary files /dev/null and b/example/demo/data/11k.data differ diff --git a/example/demo/data/12k.data b/example/demo/data/12k.data new file mode 100644 index 00000000..909e3e3e Binary files /dev/null and b/example/demo/data/12k.data differ diff --git a/example/demo/data/13k.data b/example/demo/data/13k.data new file mode 100644 index 00000000..f2a437ac Binary files /dev/null and b/example/demo/data/13k.data differ diff --git a/example/demo/data/14k.data b/example/demo/data/14k.data new file mode 100644 index 00000000..e1121ab6 Binary files /dev/null and b/example/demo/data/14k.data differ diff --git a/example/demo/data/15k.data b/example/demo/data/15k.data new file mode 100644 index 00000000..c34bcd59 Binary files /dev/null and b/example/demo/data/15k.data differ diff --git a/example/data/1M.data b/example/demo/data/1M.data similarity index 100% rename from example/data/1M.data rename to example/demo/data/1M.data diff --git a/example/data/1k.data b/example/demo/data/1k.data similarity index 100% rename from example/data/1k.data rename to example/demo/data/1k.data diff --git a/example/demo/data/20M.data b/example/demo/data/20M.data new file mode 100644 index 00000000..6c5effe7 Binary files /dev/null and b/example/demo/data/20M.data differ diff --git a/example/data/2M.data b/example/demo/data/2M.data similarity index 100% rename from example/data/2M.data rename to example/demo/data/2M.data diff --git a/example/demo/data/2k.data b/example/demo/data/2k.data new file mode 100644 index 00000000..e9784eb4 Binary files /dev/null and b/example/demo/data/2k.data differ diff --git a/example/demo/data/30M.data b/example/demo/data/30M.data new file mode 100644 index 00000000..c6fcc12d Binary files /dev/null and b/example/demo/data/30M.data differ diff --git a/example/data/3M.data b/example/demo/data/3M.data similarity index 100% rename from example/data/3M.data rename to example/demo/data/3M.data diff --git a/example/demo/data/3k.data b/example/demo/data/3k.data new file mode 100644 index 00000000..decc0c3f Binary files /dev/null and b/example/demo/data/3k.data differ diff --git a/example/data/4M.data b/example/demo/data/4M.data similarity index 100% rename from example/data/4M.data rename to example/demo/data/4M.data diff --git a/example/demo/data/4k.data b/example/demo/data/4k.data new file mode 100644 index 00000000..08e7df17 Binary files /dev/null and b/example/demo/data/4k.data differ diff --git a/example/demo/data/5M.data b/example/demo/data/5M.data new file mode 100644 index 00000000..39953167 Binary files /dev/null and b/example/demo/data/5M.data differ diff --git a/example/demo/data/5k.data b/example/demo/data/5k.data new file mode 100644 index 00000000..e7f3c2d4 Binary files /dev/null and b/example/demo/data/5k.data differ diff --git a/example/demo/data/6k.data b/example/demo/data/6k.data new file mode 100644 index 00000000..4fdf7031 Binary files /dev/null and b/example/demo/data/6k.data differ diff --git a/example/demo/data/7k.data b/example/demo/data/7k.data new file mode 100644 index 00000000..c6d6974c Binary files /dev/null and b/example/demo/data/7k.data differ diff --git a/example/demo/data/8k.data b/example/demo/data/8k.data new file mode 100644 index 00000000..6d17cf9d Binary files /dev/null and b/example/demo/data/8k.data differ diff --git a/example/demo/data/9k.data b/example/demo/data/9k.data new file mode 100644 index 00000000..ba90eb00 Binary files /dev/null and b/example/demo/data/9k.data differ diff --git a/example/demo/data/append.txt b/example/demo/data/append.txt new file mode 100644 index 00000000..240e420b --- /dev/null +++ b/example/demo/data/append.txt @@ -0,0 +1 @@ +this is a append test\! diff --git a/example/demo/data/hello.txt b/example/demo/data/hello.txt new file mode 100644 index 00000000..f2aa86d2 --- /dev/null +++ b/example/demo/data/hello.txt @@ -0,0 +1 @@ +hello diff --git a/example/demo/data/hos_upload_complete b/example/demo/data/hos_upload_complete new file mode 100755 index 00000000..045f53b2 Binary files /dev/null and b/example/demo/data/hos_upload_complete differ diff --git a/example/demo/data/hos_write_complete b/example/demo/data/hos_write_complete new file mode 100755 index 00000000..2738dc46 Binary files /dev/null and b/example/demo/data/hos_write_complete differ diff --git a/example/demo/data/test_size.sh b/example/demo/data/test_size.sh new file mode 100755 index 00000000..8051f2af --- /dev/null +++ b/example/demo/data/test_size.sh @@ -0,0 +1,20 @@ +######################################################################### +# File Name: test_times.sh +# Author: pxz +# Created Time: Mon 21 Sep 2020 04:43:35 PM CST +######################################################################### +#!/bin/bash +test_size=("1k" "10k" "100k" "1M" "2M" "3M" "4M" "5M" "10M" "20M" "30M") +#test_size=("1k" "2k" "3k" "4k" "5k" "6k" "7k" "8k" "9k" "10k" "11k" "12k" "13k" "14k" "15k") +num=0 + +#while((${num} < 15)) +while((${num} < 11)) +do + echo ./hos_write_complete mybucket ${test_size[$num]}.data 1000 + ./hos_write_complete mybucket ${test_size[$num]}.data 1000 + #echo ./hos_upload_complete mybucket ${test_size[$num]}.data 1000 + #./hos_upload_complete mybucket ${test_size[$num]}.data 1000 + let "num++" +done + diff --git a/example/demo/data/test_times.sh b/example/demo/data/test_times.sh new file mode 100755 index 00000000..a9bab6fc --- /dev/null +++ b/example/demo/data/test_times.sh @@ -0,0 +1,16 @@ +######################################################################### +# File Name: test_times.sh +# Author: pxz +# Created Time: Mon 21 Sep 2020 04:43:35 PM CST +######################################################################### +#!/bin/bash +test_times=(1, 10, 100, 1000, 10000) +num=0 + +while((${num} < 5)) +do + echo ./hos_write_complete mybucket 1k.data ${test_times[$num]} + ./hos_write_complete mybucket 1k.data ${test_times[$num]} + let "num++" +done + diff --git a/example/hos_upload_complete.cpp b/example/demo/hos_upload_complete.cpp similarity index 98% rename from example/hos_upload_complete.cpp rename to example/demo/hos_upload_complete.cpp index 84289b13..a32465a8 100644 --- a/example/hos_upload_complete.cpp +++ b/example/demo/hos_upload_complete.cpp @@ -11,7 +11,7 @@ extern "C" #include #include } -#include"../src/hos_client.h" +#include"hos_client.h" //#define test_times 10000 @@ -82,7 +82,7 @@ int main(int argc, char *argv[]) file_to_buffer(object, buf, &buf_size); debuginfo("hos_client_init start ...\n"); - hos_client_handle handle = hos_client_create("http://192.168.44.10:9098/hos/", "default", "default", 4); + hos_client_handle handle = hos_client_create("http://192.168.44.10:9098/hos/", "default", "default", 4, 3000); if (handle == NULL) { debuginfo("error:hos_client_handle\n"); diff --git a/example/hos_write_complete.cpp b/example/demo/hos_write_complete.cpp similarity index 79% rename from example/hos_write_complete.cpp rename to example/demo/hos_write_complete.cpp index ebe13fbc..615fd81f 100644 --- a/example/hos_write_complete.cpp +++ b/example/demo/hos_write_complete.cpp @@ -11,7 +11,7 @@ extern "C" #include #include } -#include"../src/hos_client.h" +#include"hos_client.h" //#define test_times 10000 @@ -71,21 +71,23 @@ int main(int argc, char *argv[]) char *object = argv[2]; int test_times = atoi(argv[3]); //int test_times = 10000; - //char *buf = (char *)malloc(1024 * 1024 * 4); - char buf[1024 * 1024 * 4]; + char *buf = (char *)malloc(1024 * 1024 * 40); + //char buf[1024 * 1024 * 4]; //char buf[1024 * 4]; size_t buf_size; int mode = FILE_MODE; - size_t fd[10000] = {0}; + size_t fd[10001] = {0}; userdata_t data = {&finished}; + //printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec); file_to_buffer(object, buf, &buf_size); debuginfo("hos_client_init start ...\n"); - hos_client_handle handle = hos_client_create("http://192.168.44.10:9098/hos/", "default", "default", 4); + hos_client_handle handle = hos_client_create("http://192.168.40.223:9098/hos/", "default", "default", 4, 4000); + //hos_client_handle handle = hos_client_create("http://192.168.32.10:9098/hos/", "default", "default", 4); if (handle == NULL) { - debuginfo("error:hos_client_handle\n"); + printf("error:hos_client_handle\n"); return -1; } debuginfo("hos_client_init success ... \n"); @@ -93,7 +95,7 @@ int main(int argc, char *argv[]) debuginfo("hos_create_bucket start ... \n"); if(hos_create_bucket(handle, bucket)) { - debuginfo("hos_create_bucket failed ... \n"); + printf("hos_create_bucket failed ... \n"); return -1; } debuginfo("hos_create_bucket success ... \n"); @@ -101,22 +103,23 @@ int main(int argc, char *argv[]) debuginfo("hos_verify_bucket start ... \n"); if(!hos_verify_bucket(handle, bucket)) { - debuginfo("hos_verify_bucket failed ... \n"); + printf("hos_verify_bucket failed ... \n"); return -1; } debuginfo("hos_verify_bucket success ... \n"); #if 1 mode = FILE_MODE; +#if 0 for (i = 0; i < test_times; i++) { - fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode); } - +#endif debuginfo("hos_upload_file start ...\n"); clock_gettime(CLOCK_MONOTONIC, &start); for (i = 0; i < test_times; i++) { + fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode); hos_write(fd[i], object, 0, 0, 0); } clock_gettime(CLOCK_MONOTONIC, &end); @@ -127,15 +130,22 @@ int main(int argc, char *argv[]) #else mode = BUFF_MODE | APPEND_MODE; +#if 0 for (i = 0; i < test_times; i++) { - fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode); } +#endif debuginfo("hos_upload_buf start ...\n"); clock_gettime(CLOCK_MONOTONIC, &start); for (i = 0; i < test_times; i++) { - hos_write(fd[i], buf, buf_size, 0, i); + fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode); + //printf("hos_write start...\n"); + if(hos_write(fd[i], buf, buf_size, 0, i + 1) != HOS_CLIENT_OK) + { + printf("error: hos_write!\n"); + } + //printf("hos_write success...\n"); } clock_gettime(CLOCK_MONOTONIC, &end); time = calc_time(start, end); @@ -147,12 +157,14 @@ int main(int argc, char *argv[]) debuginfo("hos_client_close start ...\n"); if (hos_client_destory(handle) == 0) { + //printf("start:%lu\n", start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec); + //printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec); time = calc_time(start, finished); time /= test_times; printf("hos upload finished spent %llu ns\n", time); } debuginfo("hos_client_close end ...\n"); - + free(buf); return 0; } diff --git a/example/test_size.sh b/example/demo/test_size.sh similarity index 100% rename from example/test_size.sh rename to example/demo/test_size.sh diff --git a/example/test_times.sh b/example/demo/test_times.sh similarity index 100% rename from example/test_times.sh rename to example/demo/test_times.sh diff --git a/example/performance/CMakeLists.txt b/example/performance/CMakeLists.txt new file mode 100644 index 00000000..1eea1e3a --- /dev/null +++ b/example/performance/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.5) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") +set(CMAKE_BUILD_TYPE Debug) +project(HosClientPerformance) + +link_directories(/usr/local/lib64/) +link_directories(/opt/MESA/lib/) +link_libraries(hos-client-cpp pthread) +include_directories(/opt/MESA/include) + +add_executable(HosClientPerformance HosClientPerformance.cpp) +target_link_libraries(HosClientPerformance hos-client-cpp) + diff --git a/example/performance/HosClientPerformance.cpp b/example/performance/HosClientPerformance.cpp new file mode 100644 index 00000000..a445cc75 --- /dev/null +++ b/example/performance/HosClientPerformance.cpp @@ -0,0 +1,542 @@ +/************************************************************************* + > File Name: HosClientPerformance.cpp + > Author: pxz + > Created Time: Sat 10 Oct 2020 05:26:02 PM CST + ************************************************************************/ +extern "C" +{ +#include +#include +#include +#include +#include +#include +#include +#include +} +#include"../../src/hos_client.h" + +#define MAX_THREAD_NUM 16 +#ifndef MIN +#define MIN(a,b) ((a) > (b)) ? (b) : (a) +#endif + +typedef struct conf_s +{ +#define STRING_SIZE 128 + char endpoint[STRING_SIZE]; + char bucket[STRING_SIZE]; + char object[STRING_SIZE]; + char file[STRING_SIZE]; + size_t pool_size; + size_t thread_sum; + size_t append_size; + int mode; +}conf_t; + +typedef struct thread_info_s +{ + conf_t conf; + hos_client_handle handle; + size_t thread_num; +}thread_info_t; + +static void configuration_init(conf_t *conf) +{ + strcpy(conf->endpoint, "http://192.168.40.223:9098/hos/"); + strcpy(conf->bucket, "mybucket"); + strcpy(conf->object, "myobject"); + strcpy(conf->file, "./file/test.txt"); + conf->pool_size = 4000; + conf->append_size = 1024; + conf->thread_sum = 1; + conf->mode = BUFF_MODE; + + return ; +} + +typedef struct userdata_s +{ + struct timespec *finished; +}userdata_t; + +static size_t calc_time(struct timespec start, struct timespec end) +{ + return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - + (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec)); +} + +int read_file_list(const char *path, char file_name[][256]) +{ + DIR *dir; + struct dirent *ptr; + int path_len = strlen(path); + int file_num = 0; + + if ((dir=opendir(path)) == NULL) + { + perror("Open dir error..."); + exit(-1); + } + + while (((ptr=readdir(dir)) != NULL) && (file_num < 100)) + { + if(strcmp(ptr->d_name,".")==0 || strcmp(ptr->d_name,"..")==0) ///current dir OR parrent dir + continue; + else if((ptr->d_type == 8) || (ptr->d_type == 10)) + { + memcpy(file_name[file_num], path, path_len); + strcat(file_name[file_num], ptr->d_name); + } + else if(ptr->d_type == 4) ///dir + { + continue; + } + file_num++; + } + closedir(dir); + return 0; +} + +static void callback(bool result, const char *error, void *userdata) +{ +#if 0 + userdata_t *data = (userdata_t *)userdata; + clock_gettime(CLOCK_MONOTONIC, data->finished); +#endif + return ; +} + +static int file_to_buffer(const char *file, char *buffer, size_t *len) +{ + FILE *fp = fopen(file, "r"); + int num = 0; + *len = 0; + if (fp == NULL) + { + printf("fopen file failed:%s\n", file); + return -1; + } + do{ + num = fread(&buffer[*len], 1, 4096, fp); + if (num < 0) + { + return -1; + } + *len += num; + }while(num == 4096); + fclose(fp); + return 0; +} + +static int upload_file(char *file, char *buff, int buff_len, int test_times, thread_info_t *thread_info, char *performance_info) +{ + int i; + FILE *fp = NULL; + size_t fd[3000]; + conf_t *conf = &thread_info->conf; + struct timespec tstart, tend, twrite; + size_t time_write, time_upload; + size_t len = strlen(performance_info); + char file_size[128]; + //写文件 + + clock_gettime(CLOCK_MONOTONIC, &tstart); + for (i = 0; i < test_times; i++) + { + fp = fopen(file, "w+"); + if (fp == NULL) + { + printf("error:fopen failed\n"); + return -1; + } + if (fwrite(buff, buff_len, 1, fp) != 1) + { + printf("error:fwrite failed\n"); + fclose(fp); + return -1; + } + fclose(fp); + } + clock_gettime(CLOCK_MONOTONIC, &twrite); + time_write = calc_time(tstart, twrite); + time_write /= test_times; + + //上传文件 + clock_gettime(CLOCK_MONOTONIC, &tstart); + for (i = 0; i < test_times; i++) + { + fd[i] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); + if (hos_write(fd[i], file, 0, thread_info->thread_num, 0) != HOS_CLIENT_OK) + { + printf("error:hos_write file:%s\n", file); + return -1; + } + } + clock_gettime(CLOCK_MONOTONIC, &tend); + time_upload = calc_time(tstart, tend); + time_upload /= test_times; + + sprintf(file_size, "%dk", buff_len / 1024); + sprintf(&performance_info[len], "%-20lu%-20s%-20lu%-20lu%-20lu\n", + thread_info->thread_num, file_size, time_write, time_upload, time_write + time_upload); + + //hos_close_fd(fd, thread_info->thread_num); + + return 0; +} + +static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t *thread_info, char *performance_info) +{ + int i = 0; + int j = 0; + size_t fd = 0; + size_t tmp = 0; + size_t rest = 0; + struct timespec tstart, ttmp; + size_t time = 0; + size_t len; + conf_t *conf = &thread_info->conf; + char file_size[128]; + char append_size[128]; + + fd = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); +#if 0 + size_t time_tmp = 0; + while(1) + { + tmp = j * conf->append_size; + rest = buff_len - tmp; + if (rest < conf->append_size) + { + clock_gettime(CLOCK_MONOTONIC, &tstart); + for (i = 0; i < test_times; i++) + { + hos_write(fd, &buff[tmp], rest, thread_info->thread_num, j + 1); + } + clock_gettime(CLOCK_MONOTONIC, &ttmp); + time_tmp = calc_time(tstart, ttmp); + time_tmp /= test_times; + time += time_tmp; + + sprintf(file_size, "%dk", buff_len / 1024); + sprintf(append_size, "%luk", conf->append_size / 1024); + len = strlen(performance_info); + sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20lu%-20lu\n", + thread_info->thread_num, file_size, append_size, time_tmp, time); + break; + } + clock_gettime(CLOCK_MONOTONIC, &tstart); + for (i = 0; i < test_times; i++) + { + hos_write(fd, &buff[tmp], conf->append_size, thread_info->thread_num, j + 1); + } + clock_gettime(CLOCK_MONOTONIC, &ttmp); + time_tmp = calc_time(tstart, ttmp); + time_tmp /= test_times; + time += time_tmp; + + sprintf(file_size, "%dk", buff_len / 1024); + sprintf(append_size, "%luk", conf->append_size / 1024); + len = strlen(performance_info); + sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20lu%-20lu\n", + thread_info->thread_num, file_size, append_size, time_tmp, time); + + j++; + } +#else + if (conf->mode & APPEND_MODE) + { + clock_gettime(CLOCK_MONOTONIC, &tstart); + for (i = 0; i < test_times; i++) + { + while(1) + { + tmp = j * conf->append_size; + rest = buff_len - tmp; + if (rest < conf->append_size) + { + hos_write(fd, &buff[tmp], rest, thread_info->thread_num, j + 1); + break; + } + hos_write(fd, &buff[tmp], conf->append_size, thread_info->thread_num, j + 1); + j++; + } + } + clock_gettime(CLOCK_MONOTONIC, &ttmp); + time = calc_time(tstart, ttmp); + time /= test_times; + + sprintf(file_size, "%dk", buff_len / 1024); + sprintf(append_size, "%luk", conf->append_size / 1024); + len = strlen(performance_info); + sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20d%-20lu\n", + thread_info->thread_num, file_size, append_size, 0, time); + }else + { + clock_gettime(CLOCK_MONOTONIC, &tstart); + for (i = 0; i < test_times; i++) + { + hos_write(fd, buff, buff_len, thread_info->thread_num, j + 1); + } + clock_gettime(CLOCK_MONOTONIC, &ttmp); + time = calc_time(tstart, ttmp); + time /= test_times; + + sprintf(file_size, "%dk", buff_len / 1024); + sprintf(append_size, "%luk", conf->append_size / 1024); + len = strlen(performance_info); + sprintf(&performance_info[len], "%-20lu%-20s%-20d%-20d%-20lu\n", + thread_info->thread_num, file_size, 0, 0, time); + } + + +#endif + hos_close_fd(fd, thread_info->thread_num); + + return 0; +} + +static void *put_object_thread(void *ptr) +{ + char *performance_info = NULL; + thread_info_t *thread_info = (thread_info_t *)ptr; + conf_t *conf = &thread_info->conf; + char file[128]; + size_t buff_len; + int ret; + int i; + char *buff = NULL; + char file_name[100][256]; + struct stat s_buf; + int test_times = 1; + + buff = (char *)malloc(30 * 1024 * 1024); + if (buff == NULL) + { + perror(" "); + pthread_exit(NULL); + } + + performance_info = (char *)malloc(1024 * 1024); + if (performance_info == NULL) + { + perror(" "); + free(buff); + pthread_exit(NULL); + } + memset(performance_info, 0, 10240); + memset(file_name, 0, 100 *256); + + stat(conf->file, &s_buf); + if (S_ISDIR(s_buf.st_mode)) + { + read_file_list(conf->file, file_name); + for (i = 0; i < 100; i++) + { + if (file_name[i][0] == '\0') + break; + ret = file_to_buffer(file_name[i], buff, &buff_len); + if (ret == -1) + { + free(buff); + free(performance_info); + pthread_exit(NULL); + } + if (conf->mode & BUFF_MODE) + { + upload_buff(buff, buff_len, test_times, thread_info, performance_info); + }else + { + sprintf(file, "./file/file_%lu_%d", thread_info->thread_num, i); + upload_file(file, buff, buff_len, test_times, thread_info, performance_info); + } + } + }else + { + ret = file_to_buffer(conf->file, buff, &buff_len); + if (ret == -1) + { + free(buff); + free(performance_info); + pthread_exit(NULL); + } + + if (conf->mode & BUFF_MODE) + { + upload_buff(buff, buff_len, test_times, thread_info, performance_info); + }else + { + sprintf(file, "./file/file_%lu", thread_info->thread_num); + upload_file(file, buff, buff_len, test_times, thread_info, performance_info); + } + } + free(buff); + pthread_exit(performance_info); +} + +int main(int argc, char *argv[]) +{ + int ch; + int buf_size; + conf_t conf; + char *object; + char *retval; + size_t thread_num; + size_t thread[MAX_THREAD_NUM]; + thread_info_t thread_info[MAX_THREAD_NUM]; + cpu_set_t mask; + FILE *log = NULL; + char log_name[256]; + const char *log_prefix = "./log/"; + time_t timep; +#if 0 + if (argc <= 1) + { + printf("usage: HosClientPerformance \n[-e set endpoint] \n[-b set bucket] \n" + "[-o set object] \n[-f set file] \n[-p set pool size] \n" + "[-t set thread sum] \n[-B set BUFF_MODE] \n" + "[-F set FILE_MODE] \n[-A set APPEND_MODE] \n" + "[-h show help info] \n"); + return -1; + } +#endif + configuration_init(&conf); + //读取命令行配置 + while((ch = getopt(argc, argv, "a:e:b:o:f:p:t:k:BFAh")) != -1) + { + switch(ch) + { + case 'a': + conf.append_size = 1024 * atoi(optarg); + break; + case 'e': + //endpoint + buf_size = MIN(STRING_SIZE, strlen(optarg)); + strncpy((char *)conf.endpoint, optarg, buf_size); + break; + case 'b': + buf_size = MIN(STRING_SIZE, strlen(optarg)); + strncpy((char *)conf.bucket, optarg, buf_size); + conf.bucket[buf_size] = '\0'; + break; + case 'o': + buf_size = MIN(STRING_SIZE, strlen(optarg)); + strncpy(conf.object, optarg, buf_size); + conf.object[buf_size] = '\0'; + break; + case 'f': + buf_size = MIN(STRING_SIZE, strlen(optarg)); + strncpy(conf.file, optarg, buf_size); + conf.file[buf_size] = '\0'; + break; + case 'p': + conf.pool_size = atoi(optarg); + conf.pool_size = MIN(3000, conf.pool_size); + break; + case 't': + conf.thread_sum = atoi(optarg); + break; + case 'B': + conf.mode &= BUFF_MODE; + break; + case 'F': + conf.mode &= FILE_MODE; + break; + case 'A': + conf.mode |= APPEND_MODE; + break; + case 'k': + conf.append_size = 1024 * atoi(optarg); + break; + case 'h': + default: + printf("usage: HosClientPerformance \n[-e set endpoint] \n[-b set bucket] \n" + "[-o set object] \n[-f set file] \n[-p set pool size] \n" + "[-t set thread sum] \n[-B set BUFF_MODE] \n" + "[-F set FILE_MODE] \n[-A set APPEND_MODE] \n" + "[-h show help info] \n[-a set append size(K)]\n"); + return -1; + break; + } + } + + strcpy(log_name, log_prefix); + time(&timep); + strftime(&log_name[strlen(log_prefix)], sizeof(log_name) - strlen(log_prefix),"%Y%m%d%H%M%S.log", localtime(&timep)); + log = fopen(log_name, "a+"); + if (log == NULL) + { + perror(log_name); + return -1; + } + //创建client + hos_client_handle handle = hos_client_create(conf.endpoint, "default", "default", conf.thread_sum, conf.pool_size); + if (handle == NULL) + { + printf("error:hos_client_handle\n"); + fclose(log); + return -1; + } + + //创建bucket + if (hos_create_bucket(handle, conf.bucket)) + { + printf("error:hos_create_bucket\n"); + fclose(log); + hos_client_destory(handle); + return -1; + } + + printf("\n==============================================================================================================================\n"); + if (conf.mode & BUFF_MODE) + { + printf("%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "append_size", "upload_time", "total_time"); + }else + { + printf("%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "write_time", "upload_time", "total_time"); + } + for ( thread_num = 0; thread_num < conf.thread_sum; thread_num++ ) + { + thread_info[thread_num].conf = conf; + object = thread_info[thread_num].conf.object; + sprintf(&object[strlen(object)], "-%lu", thread_num); + thread_info[thread_num].thread_num = thread_num; + thread_info[thread_num].handle = handle; + if(pthread_create(&thread[thread_num], NULL, put_object_thread, (void *)&thread_info[thread_num])) + { + perror(" "); + fclose(log); + hos_client_destory(handle); + return -1; + } + + CPU_ZERO(&mask); + CPU_SET(thread_num + 8, &mask); + if (pthread_setaffinity_np(thread[thread_num], sizeof(mask), &mask) != 0) + { + printf("warning:could not set CPU affinity, continuing...\n"); + } + } + + for (thread_num = 0; thread_num < conf.thread_sum; thread_num++) + { + pthread_join(thread[thread_num], (void **)&retval); + if (retval) + { + printf("%s", retval); + fwrite(retval, strlen(retval), 1, log); + free(retval); + } + } + + if (hos_client_destory(handle) == 0) + { + //time = calc_time(start, finished); + //time /= test_times; + //printf("hos upload finished spent %llu ns\n", time); + } + + fclose(log); + return 0; +} diff --git a/example/performance/file/100k.data b/example/performance/file/100k.data new file mode 100644 index 00000000..da1dfb90 Binary files /dev/null and b/example/performance/file/100k.data differ diff --git a/example/performance/file/10M.data b/example/performance/file/10M.data new file mode 100644 index 00000000..6c5d4031 Binary files /dev/null and b/example/performance/file/10M.data differ diff --git a/example/performance/file/10M.txt b/example/performance/file/10M.txt new file mode 100644 index 00000000..6c5d4031 Binary files /dev/null and b/example/performance/file/10M.txt differ diff --git a/example/performance/file/10k.data b/example/performance/file/10k.data new file mode 100644 index 00000000..9df64990 Binary files /dev/null and b/example/performance/file/10k.data differ diff --git a/example/performance/file/1M.data b/example/performance/file/1M.data new file mode 100644 index 00000000..9e0f96a2 Binary files /dev/null and b/example/performance/file/1M.data differ diff --git a/example/performance/file/1k.data b/example/performance/file/1k.data new file mode 100644 index 00000000..06d74050 Binary files /dev/null and b/example/performance/file/1k.data differ diff --git a/example/performance/file/20M.data b/example/performance/file/20M.data new file mode 100644 index 00000000..6c5effe7 Binary files /dev/null and b/example/performance/file/20M.data differ diff --git a/example/performance/file/2M.data b/example/performance/file/2M.data new file mode 100644 index 00000000..3301331b Binary files /dev/null and b/example/performance/file/2M.data differ diff --git a/example/performance/file/30M.data b/example/performance/file/30M.data new file mode 100644 index 00000000..c6fcc12d Binary files /dev/null and b/example/performance/file/30M.data differ diff --git a/example/performance/file/3M.data b/example/performance/file/3M.data new file mode 100644 index 00000000..b7f1f882 Binary files /dev/null and b/example/performance/file/3M.data differ diff --git a/example/performance/file/4M.data b/example/performance/file/4M.data new file mode 100644 index 00000000..98fc2c0b Binary files /dev/null and b/example/performance/file/4M.data differ diff --git a/example/performance/file/5M.data b/example/performance/file/5M.data new file mode 100644 index 00000000..39953167 Binary files /dev/null and b/example/performance/file/5M.data differ diff --git a/example/performance/file/c.sh b/example/performance/file/c.sh new file mode 100755 index 00000000..974d0741 --- /dev/null +++ b/example/performance/file/c.sh @@ -0,0 +1,15 @@ +######################################################################### +# File Name: c.sh +# Author: pxz +# Created Time: Tue 13 Oct 2020 06:28:57 PM CST +######################################################################### +#!/bin/bash +size=("1k" "10k" "100k" "1M" "2M" "3M" "4M" "5M" "10M" "20M" "30M") + +num=0 + +while((${num} < 11)) +do + truncate -s ${size[$num]} ${size[$num]}.data + let "num++" +done diff --git a/example/performance/file/file/1024k.data b/example/performance/file/file/1024k.data new file mode 100644 index 00000000..9e0f96a2 Binary files /dev/null and b/example/performance/file/file/1024k.data differ diff --git a/example/performance/file/file/1M.data b/example/performance/file/file/1M.data new file mode 100644 index 00000000..9e0f96a2 Binary files /dev/null and b/example/performance/file/file/1M.data differ diff --git a/example/performance/file/file_0 b/example/performance/file/file_0 new file mode 100644 index 00000000..06d74050 Binary files /dev/null and b/example/performance/file/file_0 differ diff --git a/example/performance/file/test.txt b/example/performance/file/test.txt new file mode 100644 index 00000000..93f84a69 Binary files /dev/null and b/example/performance/file/test.txt differ diff --git a/example/performance/test.sh b/example/performance/test.sh new file mode 100755 index 00000000..1c18bc39 --- /dev/null +++ b/example/performance/test.sh @@ -0,0 +1,17 @@ +######################################################################### +# File Name: test.sh +# Author: pxz +# Created Time: Tue 13 Oct 2020 02:28:31 PM CST +######################################################################### +#!/bin/bash +size=(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 200, 300, 400, 500) + +num=0 + +while((${num} < 37)) +do + #./HosClientPerformance -a ${size[$num]} + echo "./HosClientPerformance -a ${size[$num]} -A -f ./file/1M.data" + ./HosClientPerformance -a ${size[$num]} -A -f ./file/1M.data + let "num++" +done diff --git a/src/hos_client.cpp b/src/hos_client.cpp index f49bf9ff..b4e91867 100644 --- a/src/hos_client.cpp +++ b/src/hos_client.cpp @@ -80,14 +80,14 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client, } } -hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum) +hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum, size_t pool_size) { if (!endpoint || !accesskeyid || !secretkey || thread_sum > MAX_THREAD_NUM) { return NULL; } Aws::SDKOptions options; - //auto options = new Aws::SDKOptions; + //options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; Aws::InitAPI(options); hos_client_handle handle = (hos_client_handle)malloc(sizeof(hos_client_handle_t)); @@ -99,7 +99,7 @@ hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyi config.endpointOverride = endpoint; config.verifySSL = false; config.enableEndpointDiscovery = true; - config.executor = std::shared_ptr(std::make_shared(100));//支持线程池 + config.executor = std::shared_ptr(std::make_shared(pool_size));//支持线程池 handle->S3Client = new Aws::S3::S3Client(credentials, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false); handle->append_size = 30 * 1024 * 1024; @@ -192,7 +192,7 @@ static int hos_upload_stream(hos_client_handle handle, const char *bucket, const //设置上传数据类型 if (file_type == 0) { - if (stat(object, &buffer) == -1) + if (stat(data, &buffer) == -1) { return HOS_FILE_NOT_EXITS; } @@ -317,8 +317,7 @@ int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id } else { - //BUFF_MODE - if (stat(hos_info->object, &buffer) == -1) + if (stat(stream, &buffer) == -1) { return HOS_FILE_NOT_EXITS; } diff --git a/src/hos_client.h b/src/hos_client.h index d1cb7db5..668af9fd 100644 --- a/src/hos_client.h +++ b/src/hos_client.h @@ -80,9 +80,9 @@ typedef void (*put_finished_callback)(bool, const char *, void *); * const char *accesskeyid AWS access key ID，如”default“ * const char *secretkey AWS secret key，如”default“ * size_t thread_sum 线程总数 - * 返回值： 成功返回一个非空句柄，失败返回NULL。（失败原因都是因为输入参数为空） + * 返回值： 成功返回一个非空句柄，失败返回NULL。（失败原因都是因为输入参数不合法） *************************************************************************************/ -hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum); +hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum, size_t pool_size); /************************************************************************************* * 函数名： hos_create_bucket * 参数： hos_client_handle handle 非空句柄