diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f455372..ae44fe1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ variables: GIT_STRATEGY: "clone" BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/ INSTALL_PREFIX: "/home/mesasoft/sapp_run/" - INSTALL_DEPENDENCY_LIBRARY: framework_env libMESA_field_stat2-devel zlib-devel openssl-devel curl-devel + INSTALL_DEPENDENCY_LIBRARY: framework_env libMESA_field_stat2-devel zlib-devel openssl-devel curl-devel libMESA_prof_load libMESA_handle_logger TESTING_VERSION_BUILD: 0 stages: diff --git a/example/demo/CMakeLists.txt b/example/demo/CMakeLists.txt index ac09873b..35c548c4 100644 --- a/example/demo/CMakeLists.txt +++ b/example/demo/CMakeLists.txt @@ -1,16 +1,16 @@ cmake_minimum_required(VERSION 3.5) set(CMAKE_BUILD_TYPE Debug) -project(hos_write_complete) -project(hos_upload_complete) +project(hos_write_demo) +#project(hos_upload_complete) SET(CMAKE_BUILD_TYPE Debug) -link_directories(/usr/local/lib64/) +#link_directories(/usr/local/lib64/) link_directories(/opt/MESA/lib/) -link_libraries(hos-client-cpp) +#link_libraries(hos-client-cpp) include_directories(/opt/MESA/include) -add_executable(hos_write_complete hos_write_complete.cpp) -add_executable(hos_upload_complete hos_upload_complete.cpp) -target_link_libraries(hos_write_complete hos-client-cpp) -target_link_libraries(hos_upload_complete hos-client-cpp) +add_executable(hos_write_demo hos_write_demo.cpp) +#add_executable(hos_upload_complete hos_upload_complete.cpp) +target_link_libraries(hos_write_demo hos-client-cpp) +#target_link_libraries(hos_upload_complete hos-client-cpp) diff --git a/example/demo/conf/default.conf b/example/demo/conf/default.conf new file mode 100644 index 00000000..3d95a693 --- /dev/null +++ b/example/demo/conf/default.conf @@ -0,0 +1,16 @@ +[hos_default_conf] +hos_serverip=192.168.40.223 +hos_serverport=9098 +hos_accesskeyid="default" +hos_secretkey="default" +hos_log_path="./log/hoslog" #default +hos_log_level=30 #default +hos_poolsize=0 #default +hos_thread_sum=32 #default +hos_cache_size=102400 #default +hos_cache_count=10 #default +hos_fd_live_time_ms=1000 #default +hos_fs2_serverip=127.0.0.1 +hos_fs2_serverport=10086 +hos_fs2_path="./log/hos_fs2_log" #default +hos_fs2_format=0 #defaul \ No newline at end of file diff --git a/example/demo/hos_write_complete.cpp b/example/demo/hos_write_complete.cpp deleted file mode 100644 index 96f64b9b..00000000 --- a/example/demo/hos_write_complete.cpp +++ /dev/null @@ -1,155 +0,0 @@ -/************************************************************************* - > File Name: single_thread.cpp - > Author: pxz - > Created Time: Fri 11 Sep 2020 09:52:05 AM CST - ************************************************************************/ -extern "C" -{ -#include -#include -#include -#include -#include -} -#include"hos_client.h" - -//#define test_times 10000 - -#define debuginfo printf - -typedef struct userdata_s -{ - struct timespec *finished; -}userdata_t; - -static size_t calc_time(struct timespec start, struct timespec end) -{ - return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - - (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec)); -} - -int file_to_buffer(const char *file, char *buffer, size_t *len) -{ - FILE *fp = fopen(file, "r"); - int num = 0; - *len = 0; - if (fp == NULL) - { - debuginfo("fopen file failed:%s\n", file); - return -1; - } - do{ - num = fread(&buffer[*len], 1, 4096, fp); - if (num < 0) - { - return -1; - } - *len += num; - }while(num == 4096); - fclose(fp); - return 0; -} - -void callback(bool result, const char *error, const char *bucket, const char *object, void *userdata) -{ - userdata_t *data = (userdata_t *)userdata; - clock_gettime(CLOCK_MONOTONIC, data->finished); - return ; -} - -int main(int argc, char *argv[]) -{ - if (argc != 3) - { - debuginfo("usege: singThread [bucket name] [object name]\n"); - return -1; - } - struct timespec start, end, finished; - size_t time; - int i = 0; - char *bucket = argv[1]; - char *object = argv[2]; - char *buf = (char *)malloc(1024 * 1024 * 40); - size_t buf_size; - int mode = FILE_MODE; - size_t fd[10001] = {0}; - userdata_t data = {&finished}; - - file_to_buffer(object, buf, &buf_size); - - debuginfo("hos_client_init start ...\n"); - hos_client_handle handle = hos_client_create("192.168.44.12", 9098, "default", "default", 100); - //hos_client_handle handle = hos_client_create("http://192.168.32.10:9098/hos/", "default", "default", 4); - if (handle == NULL) - { - printf("error:hos_client_handle\n"); - return -1; - } - debuginfo("hos_client_init success ... \n"); - - debuginfo("hos_create_bucket start ... \n"); - if(hos_create_bucket(handle, bucket)) - { - printf("hos_create_bucket failed ... \n"); - return -1; - } - debuginfo("hos_create_bucket success ... \n"); - - debuginfo("hos_verify_bucket start ... \n"); - if(!hos_verify_bucket(handle, bucket)) - { - printf("hos_verify_bucket failed ... \n"); - return -1; - } - debuginfo("hos_verify_bucket success ... \n"); - -#if 1 - mode = FILE_MODE; - debuginfo("hos_upload_file start ...\n"); - clock_gettime(CLOCK_MONOTONIC, &start); - fd[0] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode); - hos_write(fd[0], object, 0, 0); - clock_gettime(CLOCK_MONOTONIC, &end); - time = calc_time(start, end); - printf("hos_upload_file spent %llu ns\n", time); - debuginfo("hos_upload_file end ...\n"); -#else - - mode = BUFF_MODE | APPEND_MODE; -#if 0 - for (i = 0; i < test_times; i++) - { - } -#endif - debuginfo("hos_upload_buf start ...\n"); - clock_gettime(CLOCK_MONOTONIC, &start); - for (i = 0; i < test_times; i++) - { - fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode); - //printf("hos_write start...\n"); - if(hos_write(fd[i], buf, buf_size, 0, i + 1) != HOS_CLIENT_OK) - { - printf("error: hos_write!\n"); - } - //printf("hos_write success...\n"); - } - clock_gettime(CLOCK_MONOTONIC, &end); - time = calc_time(start, end); - time /= test_times; - printf("hos_upload_buf spent %llu ns\n", time); - debuginfo("hos_upload_buf end ...\n"); - -#endif - debuginfo("hos_client_close start ...\n"); - if (hos_client_destory(handle) == 0) - { - //printf("start:%lu\n", start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec); - //printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec); - time = calc_time(start, finished); - printf("hos upload finished spent %llu ns\n", time); - } - - debuginfo("hos_client_close end ...\n"); - free(buf); - return 0; -} diff --git a/example/demo/hos_write_demo.cpp b/example/demo/hos_write_demo.cpp new file mode 100644 index 00000000..9a8ecb1f --- /dev/null +++ b/example/demo/hos_write_demo.cpp @@ -0,0 +1,155 @@ +/************************************************************************* + > File Name: single_thread.cpp + > Author: pxz + > Created Time: Fri 11 Sep 2020 09:52:05 AM CST + ************************************************************************/ +extern "C" +{ +#include +#include +#include +#include +#include +#include +} +#include"hos_client.h" + +//#define test_times 10000 + +typedef struct userdata_s +{ + struct timespec *finished; +}userdata_t; + +static size_t calc_time(struct timespec start, struct timespec end) +{ + return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - + (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec)); +} + +int file_to_buffer(const char *file, char *buffer, size_t *len) +{ + FILE *fp = fopen(file, "r"); + int num = 0; + *len = 0; + if (fp == NULL) + { + printf("fopen file failed:%s\n", file); + return -1; + } + do{ + num = fread(&buffer[*len], 1, 4096, fp); + if (num < 0) + { + return -1; + } + *len += num; + }while(num == 4096); + fclose(fp); + return 0; +} + +void callback(bool result, const char *error, const char *bucket, const char *object, void *userdata) +{ + userdata_t *data = (userdata_t *)userdata; + clock_gettime(CLOCK_MONOTONIC, data->finished); + return ; +} + +int main(int argc, char *argv[]) +{ + if (argc != 4) + { + printf("usege: [conf file] [module name] [file name]\n"); + return -1; + } + struct timespec start, end, finished; + size_t time; + int i = 0; + char *conf_file = argv[1]; + char *module_name = argv[2]; + char *file_name = argv[3]; + struct stat buffer; + char *buf = NULL; + size_t buf_size; + int mode = FILE_MODE; + size_t fd[10001] = {0}; + userdata_t data = {&finished}; + hos_instance hos_instance = NULL; + char object[1024]; + + if (stat(file_name, &buffer) == -1) + { + printf("%s not exits.", conf_file); + return -1; + } + + buf = (char *)calloc(1, buffer.st_size); + + if (file_to_buffer(file_name, buf, &buf_size) == -1) + { + free(buf); + return -1; + } + + printf("hos_init_instance start ...\n"); + hos_instance = hos_get_instance(); + if (hos_instance->result == false) + { + hos_instance = hos_init_instance(conf_file, module_name, 1, "hos_test_bucket"); + } + if (hos_instance->result == false) + { + printf("error:hos_init_instance\n"); + printf("error:%s", hos_instance->error_message); + return -1; + } + printf("hos_init_instance success ... \n"); + + mode = FILE_MODE; + printf("hos_write file start ...\n"); + snprintf(object, 1023, "%s_write_file", file_name); + fd[0] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode); + if (hos_write(fd[i], file_name, 0, 0) != HOS_CLIENT_OK) + { + printf("error: hos_write fialed!\n"); + } + hos_close_fd(fd[1], 0); + printf("hos_write file end ...\n"); + + mode = BUFF_MODE; + printf("hos_write buff start ...\n"); + snprintf(object, 1023, "%s_write_buff", file_name); + fd[1] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode); + if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK) + { + printf("error: hos_write failed!\n"); + } + hos_close_fd(fd[1], 0); + printf("hos_write buff end ...\n"); + + mode = BUFF_MODE | APPEND_MODE; + printf("hos_write buff start ...\n"); + snprintf(object, 1023, "%s_write_APPEND", file_name); + fd[2] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode); + if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK) + { + printf("error: hos_write failed 1st!\n"); + } + if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK) + { + printf("error: hos_write failed 2nd!\n"); + } + hos_close_fd(fd[2], 0); + printf("hos_write buff end ...\n"); + + printf("hos_shutdown_instance start ...\n"); + if (hos_shutdown_instance() != 0) + { + printf("error: hos_shutdown_instance fialed\n"); + } + printf("hos_shutdown_instance end ...\n"); + + free(buf); + return 0; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f6114724..fea7ce45 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,12 +9,17 @@ set(CMKAE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC") add_library(${lib_name}_shared SHARED hos_client.cpp hos_hash.cpp) target_link_libraries(${lib_name}_shared "-Wl,--whole-archive" + libaws-c-common.a + libaws-checksums.a + libaws-c-event-stream.a libaws-cpp-sdk-core.a libaws-cpp-sdk-s3.a "-Wl,--no-whole-archive" - libMESA_field_stat2.so libcurl.so - libpthread.so) + libpthread.so + libMESA_prof_load.so + libMESA_field_stat2.so + libMESA_handle_logger.so) set_target_properties(${lib_name}_shared PROPERTIES OUTPUT_NAME ${lib_name}) #for ASAN diff --git a/src/hos_client.cpp b/src/hos_client.cpp index 303aaaf4..4d5dbd4d 100644 --- a/src/hos_client.cpp +++ b/src/hos_client.cpp @@ -237,6 +237,13 @@ static void hos_client_create() hos_config_t *hos_conf = &g_hos_handle.hos_config; void *log = g_hos_handle.log; + if (g_hos_handle.S3Client != NULL) + { + g_hos_handle.count++; + g_hos_instance.result = true; + return ; + } + Aws::InitAPI(g_options); Aws::Client::ClientConfiguration config; Aws::Auth::AWSCredentials credentials(hos_conf->accesskeyid, hos_conf->secretkey); @@ -626,6 +633,18 @@ static bool hos_putobject_sync(Aws::S3::Model::PutObjectRequest& request, size_t } } +hos_instance hos_get_instance() +{ + if (g_hos_handle.S3Client != NULL) + { + g_hos_handle.count++; + g_hos_instance.result = true; + return &g_hos_instance; + } + g_hos_instance.result = false; + return &g_hos_instance; +} + hos_instance hos_init_instance(const char *conf_path, const char *module, size_t thread_num, const char *bucket) { hos_config_t *hos_conf = &g_hos_handle.hos_config; @@ -650,12 +669,13 @@ hos_instance hos_init_instance(const char *conf_path, const char *module, size_t MESA_load_profile_uint_def(conf_path, module, "hos_cache_size", &hos_conf->cache_size, 102400); MESA_load_profile_uint_def(conf_path, module, "hos_cache_count", &hos_conf->cache_count, 10); MESA_load_profile_uint_def(conf_path, module, "hos_fd_live_time_ms", &hos_conf->timeout, 1000); - MESA_load_profile_string_nodef(conf_path, module, "hos_fs2_serverip", hos_conf->fs2_ip, MAX_HOS_STRING_LEN); + MESA_load_profile_string_nodef(conf_path, module, "hos_fs2_serverip", hos_conf->fs2_ip, INET6_ADDRSTRLEN); MESA_load_profile_uint_nodef(conf_path, module, "hos_fs2_serverport", &hos_conf->fs2_port); MESA_load_profile_string_def(conf_path, module, "hos_fs2_path", hos_conf->fs2_path, sizeof(hos_conf->fs2_path), "./hos_fs2.stat"); MESA_load_profile_uint_def(conf_path, module, "hos_fs2_format", &hos_conf->fs2_fmt, 0); - if (hos_conf->ip && hos_conf->port && hos_conf->accesskeyid && hos_conf->secretkey) + if (hos_conf->ip && hos_conf->port && strlen(hos_conf->accesskeyid) && strlen(hos_conf->secretkey)) { + MESA_handle_runtime_log_creation("./log"); g_hos_handle.log = MESA_create_runtime_log_handle(hos_conf->log_path, hos_conf->log_level); if (log == NULL) { @@ -1092,10 +1112,7 @@ int hos_close_fd(size_t fd, size_t thread_id) hos_fd_context_t *a_fd_context = NULL; char num[128]; hos_config_t *hos_conf = &g_hos_handle.hos_config; -<<<<<<< HEAD size_t upload_len = 0; -======= ->>>>>>> c9c751ab... 增加同步模式 if (fd < 3 || thread_id > hos_conf->thread_num) { @@ -1140,19 +1157,11 @@ int hos_close_fd(size_t fd, size_t thread_id) if (hos_conf->pool_thread_size > 0) { -<<<<<<< HEAD hos_putobject_async(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object); } else { hos_putobject_sync(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object); -======= - hos_putobject_async(request, a_fd_context->cache->gcount(), thread_id, fd, a_fd_context->bucket, a_fd_context->object); - } - else - { - hos_putobject_sync(request, a_fd_context->cache->gcount(), thread_id, fd, a_fd_context->bucket, a_fd_context->object); ->>>>>>> c9c751ab... 增加同步模式 } } } diff --git a/src/hos_client.h b/src/hos_client.h index 657a51c2..03ae838c 100644 --- a/src/hos_client.h +++ b/src/hos_client.h @@ -105,11 +105,10 @@ hos_instance hos_client_create(const char *serverip, size_t port, const char *ac *************************************************************************************/ hos_instance hos_init_instance(const char *conf_path, const char *module, size_t thread_num, const char *bucket); /************************************************************************************* - * 函数名： hos_create_bucket - * 参数: const char *bucket 桶名称 - * 返回值： bool 成功返回true，失败返回false + * 函数名： hos_get_instance + * 返回值： hos_instance 成功，result 为true *************************************************************************************/ -//bool hos_verify_bucket(const char *bucket); +hos_instance hos_get_instance(); /************************************************************************************* * 函数名： hos_create_bucket * 参数： const char * bucket 桶名称 diff --git a/support/CMakeLists.txt b/support/CMakeLists.txt index 62652cd5..a094c04b 100644 --- a/support/CMakeLists.txt +++ b/support/CMakeLists.txt @@ -2,7 +2,7 @@ include(ExternalProject) set(AWSS3_ROOT ${CMAKE_CURRENT_BINARY_DIR}) set(AWSS3_URL ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-master.zip) -set(AWSS3_URL_MD5 7b89ae5c7c7c01bfe582ca5fbad728de) +set(AWSS3_URL_MD5 a38984c137d7768dec2a7cde02874c32) set(AWSS3_CONFIGURE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && cmake . -DBUILD_ONLY=s3) set(AWSS3_MAKE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make) set(AWSS3_INSTALL cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make install PREFIX=${SUPPORT_INSTALL_PREFIX}) diff --git a/support/aws-sdk-cpp-master.zip b/support/aws-sdk-cpp-master.zip index a0f3edcf..1d0d8ae8 100644 Binary files a/support/aws-sdk-cpp-master.zip and b/support/aws-sdk-cpp-master.zip differ