增加测试客户端程序

.gitlab-ci.yml

@@ -3,7 +3,7 @@ variables:
   GIT_STRATEGY: "clone"
   BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
   INSTALL_PREFIX: "/home/mesasoft/sapp_run/"
-  INSTALL_DEPENDENCY_LIBRARY: framework_env libMESA_field_stat2-devel zlib-devel openssl-devel curl-devel
+  INSTALL_DEPENDENCY_LIBRARY: framework_env libMESA_field_stat2-devel zlib-devel openssl-devel curl-devel libMESA_prof_load libMESA_handle_logger
   TESTING_VERSION_BUILD: 0
 
 stages:

example/demo/CMakeLists.txt

@@ -1,16 +1,16 @@
 cmake_minimum_required(VERSION 3.5)
 set(CMAKE_BUILD_TYPE Debug)
-project(hos_write_complete)
+project(hos_write_demo)
-project(hos_upload_complete)
+#project(hos_upload_complete)
 
 SET(CMAKE_BUILD_TYPE Debug)
-link_directories(/usr/local/lib64/)
+#link_directories(/usr/local/lib64/)
 link_directories(/opt/MESA/lib/)
-link_libraries(hos-client-cpp)
+#link_libraries(hos-client-cpp)
 include_directories(/opt/MESA/include)
 
-add_executable(hos_write_complete hos_write_complete.cpp)
+add_executable(hos_write_demo hos_write_demo.cpp)
-add_executable(hos_upload_complete hos_upload_complete.cpp)
+#add_executable(hos_upload_complete hos_upload_complete.cpp)
-target_link_libraries(hos_write_complete hos-client-cpp)
+target_link_libraries(hos_write_demo hos-client-cpp)
-target_link_libraries(hos_upload_complete hos-client-cpp)
+#target_link_libraries(hos_upload_complete hos-client-cpp)
 

example/demo/conf/default.conf

@@ -0,0 +1,16 @@
+[hos_default_conf]
+hos_serverip=192.168.40.223
+hos_serverport=9098
+hos_accesskeyid="default"
+hos_secretkey="default"
+hos_log_path="./log/hoslog" #default
+hos_log_level=30 #default
+hos_poolsize=0 #default
+hos_thread_sum=32 #default
+hos_cache_size=102400 #default
+hos_cache_count=10 #default
+hos_fd_live_time_ms=1000 #default
+hos_fs2_serverip=127.0.0.1
+hos_fs2_serverport=10086
+hos_fs2_path="./log/hos_fs2_log" #default
+hos_fs2_format=0 #defaul

example/demo/hos_write_complete.cpp

@@ -1,155 +0,0 @@
-/*************************************************************************
-    > File Name: single_thread.cpp
-    > Author: pxz
-    > Created Time: Fri 11 Sep 2020 09:52:05 AM CST
- ************************************************************************/
-extern "C"
-{
-#include<stdio.h>
-#include<stdlib.h>
-#include<unistd.h>
-#include<string.h>
-#include<time.h>
-}
-#include"hos_client.h"
-
-//#define test_times 10000
-
-#define debuginfo printf
-
-typedef struct userdata_s
-{
-    struct timespec *finished;
-}userdata_t;
-
-static size_t  calc_time(struct timespec start, struct timespec end)
-{
-    return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - 
-        (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
-}
-
-int file_to_buffer(const char *file, char *buffer, size_t *len)
-{
-    FILE *fp = fopen(file, "r");
-    int num = 0;
-    *len = 0;
-    if (fp == NULL)
-    {
-        debuginfo("fopen file failed:%s\n", file);
-        return -1;
-    }
-    do{
-        num = fread(&buffer[*len], 1, 4096, fp);
-        if (num < 0)
-        {
-            return -1;
-        }
-        *len += num;
-    }while(num == 4096);
-    fclose(fp);
-    return 0;
-}
-
-void callback(bool result, const char *error, const char *bucket, const char *object, void *userdata)
-{
-    userdata_t *data = (userdata_t *)userdata;
-    clock_gettime(CLOCK_MONOTONIC, data->finished);
-    return ;
-}
-
-int main(int argc, char *argv[])
-{
-    if (argc != 3)
-    {
-        debuginfo("usege: singThread [bucket name] [object name]\n");
-        return -1;
-    }
-    struct timespec start, end, finished;
-    size_t time;
-    int i = 0;
-    char *bucket = argv[1];
-    char *object = argv[2];
-    char *buf = (char *)malloc(1024 * 1024 * 40);
-    size_t buf_size;
-    int mode = FILE_MODE;
-    size_t fd[10001] = {0}; 
-    userdata_t data = {&finished};
-
-    file_to_buffer(object, buf, &buf_size);
-
-    debuginfo("hos_client_init start ...\n");
-    hos_client_handle handle = hos_client_create("192.168.44.12", 9098, "default", "default", 100);
-    //hos_client_handle handle = hos_client_create("http://192.168.32.10:9098/hos/", "default", "default", 4);
-    if (handle == NULL) 
-    {
-        printf("error:hos_client_handle\n");
-        return -1;
-    }  
-    debuginfo("hos_client_init success ... \n");
-
-    debuginfo("hos_create_bucket start ... \n");
-    if(hos_create_bucket(handle, bucket))
-    {
-        printf("hos_create_bucket failed ... \n");
-        return -1;
-    }
-    debuginfo("hos_create_bucket success ... \n");
-
-    debuginfo("hos_verify_bucket start ... \n");
-    if(!hos_verify_bucket(handle, bucket))
-    {
-        printf("hos_verify_bucket failed ... \n");
-        return -1;
-    }
-    debuginfo("hos_verify_bucket success ... \n");
-    
-#if 1
-    mode = FILE_MODE;
-    debuginfo("hos_upload_file start ...\n");
-    clock_gettime(CLOCK_MONOTONIC, &start);
-    fd[0] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
-    hos_write(fd[0], object, 0, 0);
-    clock_gettime(CLOCK_MONOTONIC, &end);
-    time = calc_time(start, end);
-    printf("hos_upload_file spent %llu ns\n", time);
-    debuginfo("hos_upload_file end ...\n");
-#else
-
-    mode = BUFF_MODE | APPEND_MODE;
-#if 0
-    for (i = 0; i < test_times; i++)
-    {
-    }
-#endif
-    debuginfo("hos_upload_buf start ...\n");
-    clock_gettime(CLOCK_MONOTONIC, &start);
-    for (i = 0; i < test_times; i++)
-    {
-        fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
-        //printf("hos_write start...\n");
-        if(hos_write(fd[i], buf, buf_size, 0, i + 1) != HOS_CLIENT_OK)
-        {
-            printf("error: hos_write!\n");
-        }
-        //printf("hos_write success...\n");
-    }
-    clock_gettime(CLOCK_MONOTONIC, &end);
-    time = calc_time(start, end);
-    time /= test_times;
-    printf("hos_upload_buf spent %llu ns\n", time);
-    debuginfo("hos_upload_buf end ...\n");
-
-#endif
-    debuginfo("hos_client_close start ...\n");
-    if (hos_client_destory(handle) == 0)
-    {
-        //printf("start:%lu\n", start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec);
-        //printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec);
-        time = calc_time(start, finished);
-        printf("hos upload finished spent %llu ns\n", time);
-    }
-
-    debuginfo("hos_client_close end ...\n");
-    free(buf);
-    return 0;
-}

example/demo/hos_write_demo.cpp

@@ -0,0 +1,155 @@
+/*************************************************************************
+    > File Name: single_thread.cpp
+    > Author: pxz
+    > Created Time: Fri 11 Sep 2020 09:52:05 AM CST
+ ************************************************************************/
+extern "C"
+{
+#include<stdio.h>
+#include<stdlib.h>
+#include<unistd.h>
+#include<string.h>
+#include<time.h>
+#include<sys/stat.h>
+}
+#include"hos_client.h"
+
+//#define test_times 10000
+
+typedef struct userdata_s
+{
+    struct timespec *finished;
+}userdata_t;
+
+static size_t  calc_time(struct timespec start, struct timespec end)
+{
+    return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - 
+        (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
+}
+
+int file_to_buffer(const char *file, char *buffer, size_t *len)
+{
+    FILE *fp = fopen(file, "r");
+    int num = 0;
+    *len = 0;
+    if (fp == NULL)
+    {
+        printf("fopen file failed:%s\n", file);
+        return -1;
+    }
+    do{
+        num = fread(&buffer[*len], 1, 4096, fp);
+        if (num < 0)
+        {
+            return -1;
+        }
+        *len += num;
+    }while(num == 4096);
+    fclose(fp);
+    return 0;
+}
+
+void callback(bool result, const char *error, const char *bucket, const char *object, void *userdata)
+{
+    userdata_t *data = (userdata_t *)userdata;
+    clock_gettime(CLOCK_MONOTONIC, data->finished);
+    return ;
+}
+
+int main(int argc, char *argv[])
+{
+    if (argc != 4)
+    {
+        printf("usege: [conf file] [module name] [file name]\n");
+        return -1;
+    }
+    struct timespec start, end, finished;
+    size_t time;
+    int i = 0;
+    char *conf_file = argv[1];
+    char *module_name = argv[2];
+    char *file_name = argv[3];
+    struct stat buffer;    
+    char *buf = NULL;
+    size_t buf_size;
+    int mode = FILE_MODE;
+    size_t fd[10001] = {0}; 
+    userdata_t data = {&finished};
+    hos_instance hos_instance = NULL;
+    char object[1024];
+
+    if (stat(file_name, &buffer) == -1)
+    {
+        printf("%s not exits.", conf_file);
+        return -1;
+    }
+    
+    buf = (char *)calloc(1, buffer.st_size);
+
+    if (file_to_buffer(file_name, buf, &buf_size) == -1)
+    {
+        free(buf);
+        return -1;
+    }
+
+    printf("hos_init_instance start ...\n");
+    hos_instance = hos_get_instance();
+    if (hos_instance->result == false)
+    {
+        hos_instance = hos_init_instance(conf_file, module_name, 1, "hos_test_bucket");
+    }
+    if (hos_instance->result == false) 
+    {
+        printf("error:hos_init_instance\n");
+        printf("error:%s", hos_instance->error_message);
+        return -1;
+    }  
+    printf("hos_init_instance success ... \n");
+
+    mode = FILE_MODE;
+    printf("hos_write file start ...\n");
+    snprintf(object, 1023, "%s_write_file", file_name);
+    fd[0] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode);
+    if (hos_write(fd[i], file_name, 0, 0) != HOS_CLIENT_OK)
+    {
+        printf("error: hos_write fialed!\n");
+    }
+    hos_close_fd(fd[1], 0);
+    printf("hos_write file end ...\n");
+
+    mode = BUFF_MODE;
+    printf("hos_write buff start ...\n");
+    snprintf(object, 1023, "%s_write_buff", file_name);
+    fd[1] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode);
+    if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK)
+    {
+        printf("error: hos_write failed!\n");
+    }
+    hos_close_fd(fd[1], 0);
+    printf("hos_write buff end ...\n");
+
+    mode = BUFF_MODE | APPEND_MODE;
+    printf("hos_write buff start ...\n");
+    snprintf(object, 1023, "%s_write_APPEND", file_name);
+    fd[2] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode);
+    if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK)
+    {
+        printf("error: hos_write failed 1st!\n");
+    }
+    if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK)
+    {
+        printf("error: hos_write failed 2nd!\n");
+    }
+    hos_close_fd(fd[2], 0);
+    printf("hos_write buff end ...\n");
+
+    printf("hos_shutdown_instance start ...\n");
+    if (hos_shutdown_instance() != 0)
+    {
+        printf("error: hos_shutdown_instance fialed\n");
+    }
+    printf("hos_shutdown_instance end ...\n");
+    
+    free(buf);
+    return 0;
+}

src/CMakeLists.txt

@@ -9,12 +9,17 @@ set(CMKAE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC")
 add_library(${lib_name}_shared SHARED hos_client.cpp hos_hash.cpp)
 target_link_libraries(${lib_name}_shared
     "-Wl,--whole-archive"
+    libaws-c-common.a
+    libaws-checksums.a
+    libaws-c-event-stream.a
     libaws-cpp-sdk-core.a 
     libaws-cpp-sdk-s3.a
     "-Wl,--no-whole-archive"
-    libMESA_field_stat2.so
     libcurl.so
-    libpthread.so)
+    libpthread.so
+    libMESA_prof_load.so
+    libMESA_field_stat2.so
+    libMESA_handle_logger.so)
 set_target_properties(${lib_name}_shared PROPERTIES OUTPUT_NAME ${lib_name})
 
 #for ASAN

src/hos_client.cpp

@@ -237,6 +237,13 @@ static void hos_client_create()
     hos_config_t *hos_conf = &g_hos_handle.hos_config;
     void *log = g_hos_handle.log;
 
+    if (g_hos_handle.S3Client != NULL)
+    {
+        g_hos_handle.count++;
+        g_hos_instance.result = true;
+        return ;
+    }
+
     Aws::InitAPI(g_options);
     Aws::Client::ClientConfiguration config;
     Aws::Auth::AWSCredentials credentials(hos_conf->accesskeyid, hos_conf->secretkey);
@@ -626,6 +633,18 @@ static bool hos_putobject_sync(Aws::S3::Model::PutObjectRequest& request, size_t
     }
 }
 
+hos_instance hos_get_instance()
+{
+    if (g_hos_handle.S3Client != NULL)
+    {
+        g_hos_handle.count++;
+        g_hos_instance.result = true;
+        return &g_hos_instance;
+    }
+    g_hos_instance.result = false;
+    return &g_hos_instance;
+}
+
 hos_instance hos_init_instance(const char *conf_path, const char *module, size_t thread_num, const char *bucket)
 {
     hos_config_t *hos_conf = &g_hos_handle.hos_config;
@@ -650,12 +669,13 @@ hos_instance hos_init_instance(const char *conf_path, const char *module, size_t
     MESA_load_profile_uint_def(conf_path, module, "hos_cache_size", &hos_conf->cache_size, 102400);
     MESA_load_profile_uint_def(conf_path, module, "hos_cache_count", &hos_conf->cache_count, 10);
     MESA_load_profile_uint_def(conf_path, module, "hos_fd_live_time_ms", &hos_conf->timeout, 1000);
-    MESA_load_profile_string_nodef(conf_path, module, "hos_fs2_serverip", hos_conf->fs2_ip, MAX_HOS_STRING_LEN);
+    MESA_load_profile_string_nodef(conf_path, module, "hos_fs2_serverip", hos_conf->fs2_ip, INET6_ADDRSTRLEN);
     MESA_load_profile_uint_nodef(conf_path, module, "hos_fs2_serverport", &hos_conf->fs2_port);
     MESA_load_profile_string_def(conf_path, module, "hos_fs2_path", hos_conf->fs2_path, sizeof(hos_conf->fs2_path), "./hos_fs2.stat");
     MESA_load_profile_uint_def(conf_path, module, "hos_fs2_format", &hos_conf->fs2_fmt, 0);
-    if (hos_conf->ip && hos_conf->port && hos_conf->accesskeyid && hos_conf->secretkey)
+    if (hos_conf->ip && hos_conf->port && strlen(hos_conf->accesskeyid) && strlen(hos_conf->secretkey))
     {
+        MESA_handle_runtime_log_creation("./log");
         g_hos_handle.log = MESA_create_runtime_log_handle(hos_conf->log_path, hos_conf->log_level);
         if (log == NULL)
         {
@@ -1092,10 +1112,7 @@ int hos_close_fd(size_t fd, size_t thread_id)
     hos_fd_context_t *a_fd_context = NULL;
     char num[128];
     hos_config_t *hos_conf = &g_hos_handle.hos_config;
-<<<<<<< HEAD
     size_t upload_len = 0;
-=======
->>>>>>> c9c751ab... 增加同步模式
 
     if (fd < 3 || thread_id > hos_conf->thread_num)
     {
@@ -1140,19 +1157,11 @@ int hos_close_fd(size_t fd, size_t thread_id)
 
             if (hos_conf->pool_thread_size > 0)
             {
-<<<<<<< HEAD
                 hos_putobject_async(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object);
             }
             else
             {
                 hos_putobject_sync(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object);
-=======
-                hos_putobject_async(request, a_fd_context->cache->gcount(), thread_id, fd, a_fd_context->bucket, a_fd_context->object);
-            }
-            else
-            {
-                hos_putobject_sync(request, a_fd_context->cache->gcount(), thread_id, fd, a_fd_context->bucket, a_fd_context->object);
->>>>>>> c9c751ab... 增加同步模式
             }
         }
     }

src/hos_client.h

@@ -105,11 +105,10 @@ hos_instance hos_client_create(const char *serverip, size_t port, const char *ac
 *************************************************************************************/
 hos_instance hos_init_instance(const char *conf_path, const char *module, size_t thread_num, const char *bucket);
 /*************************************************************************************
- * 函数名： hos_create_bucket
+ * 函数名： hos_get_instance
- * 参数:    const char *bucket  桶名称
+ * 返回值： hos_instance 成功，result 为true
- * 返回值： bool 成功返回true，失败返回false
 *************************************************************************************/
-//bool hos_verify_bucket(const char *bucket);
+hos_instance hos_get_instance();
 /*************************************************************************************
  * 函数名： hos_create_bucket
  * 参数：   const char * bucket         桶名称

support/CMakeLists.txt

@@ -2,7 +2,7 @@ include(ExternalProject)
 
 set(AWSS3_ROOT            ${CMAKE_CURRENT_BINARY_DIR})
 set(AWSS3_URL             ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-master.zip)
-set(AWSS3_URL_MD5         7b89ae5c7c7c01bfe582ca5fbad728de)
+set(AWSS3_URL_MD5         a38984c137d7768dec2a7cde02874c32)
 set(AWSS3_CONFIGURE       cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && cmake . -DBUILD_ONLY=s3)
 set(AWSS3_MAKE            cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make) 
 set(AWSS3_INSTALL         cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make install PREFIX=${SUPPORT_INSTALL_PREFIX})