Develop 09.22

2020-09-22 17:22:21 +08:00
parent eb41917cb2
commit 10e17cb043
7 changed files with 58 additions and 45 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,21 +4,22 @@ set(lib_name hos-client-cpp)
 project(${lib_name})

 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
-include(Version)
+#include(Version)

 set(CMAKE_MACOSX_RPATH 0)

 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")

-set(CMAKE_INSTALL_PREFIX /opt/MESA/lib)
+set(CMAKE_INSTALL_PREFIX /opt/MESA)
 set(SUPPORT_INSTALL_PREFIX /usr/local/lib64)

 add_subdirectory(support) 
 add_subdirectory(src) 

-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhso-client-cpp.so DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT PROFILE)
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhos-client-cpp.a DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT PROFILE)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhos-client-cpp.so DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT PROFILE)
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/src/hos_client.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT PROFILE)
+#install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhos-client-cpp.a DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT PROFILE)

-include(Package)
+#include(Package)

--- a/example/CMakeLists.txt
+++ b/example/CMakeLists.txt
@@ -5,9 +5,9 @@ project(singleThread)
 SET(CMAKE_BUILD_TYPE Debug)
 link_directories(/usr/local/lib64/)
 link_directories(/opt/MESA/lib/)
-link_libraries(hos_client_cpp)
-#set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR})
-add_executable(singleThread single_thread.cpp)
-target_link_libraries(singleThread hos_client_cpp)
+link_libraries(hos-client-cpp)
+include_directories(/opt/MESA/include)
+
+add_executable(singleThread single_thread.cpp)
+target_link_libraries(singleThread hos-client-cpp)

-#install(TARGETS singleThread DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})
--- a/example/single_thread.cpp
+++ b/example/single_thread.cpp
@@ -17,6 +17,11 @@ extern "C"

 #define debuginfo (void)

+typedef struct userdata_s
+{
+    struct timespec *finished;
+}userdata_t;
+
 static size_t  calc_time(struct timespec start, struct timespec end)
 {
    return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - 
@@ -47,12 +52,8 @@ int file_to_buffer(const char *file, char *buffer, size_t *len)

 void callback(bool result, const char *error, void *userdata)
 {
-    //debuginfo("result : %s\n", result ? "true":"false");
-    if (result)
-        return ;
-    //debuginfo("error: %s\n", error);
-    //debuginfo("userdata:%s\n", (char *)userdata);
-    hos_close_fd(*(int *)userdata, 0);
+    userdata_t *data = (userdata_t *)userdata;
+    clock_gettime(CLOCK_MONOTONIC, data->finished);
    return ;
 }

@@ -63,7 +64,7 @@ int main(int argc, char *argv[])
        debuginfo("usege: singThread [bucket name] [object name]\n");
        return -1;
    }
-    struct timespec start, end;
+    struct timespec start, end, finished;
    size_t time;
    int i = 0;
    char *bucket = argv[1];
@@ -73,11 +74,14 @@ int main(int argc, char *argv[])
    //char *buf = (char *)malloc(1024 * 1024 * 4);
    char buf[1024 * 1024 * 4];
    size_t buf_size;
+    int mode = FILE_MODE;
+    size_t fd = 0; 
+    userdata_t data = {&finished};

    file_to_buffer(object, buf, &buf_size);

    debuginfo("hos_client_init start ...\n");
-    hos_client_handle handle = hos_client_create("http://192.168.44.12:9098/hos/", "default", "default", 4);
+    hos_client_handle handle = hos_client_create("http://192.168.44.10:9098/hos/", "default", "default", 4);
    if (handle == NULL) 
    {
        debuginfo("error:hos_client_handle\n");
@@ -101,10 +105,8 @@ int main(int argc, char *argv[])
    }
    debuginfo("hos_verify_bucket success ... \n");
    
-#if 1 
-    int mode = FILE_MODE;
-    size_t fd = 0; 
-    fd = hos_open_fd(handle, bucket, object, callback, (void *)&fd, 0, mode);
+#if 0
+    fd = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
    debuginfo("hos_upload_file start ...\n");
    clock_gettime(CLOCK_MONOTONIC, &start);
    for (i = 0; i < test_times; i++)
@@ -118,9 +120,9 @@ int main(int argc, char *argv[])
    debuginfo("hos_upload_file end ...\n");
 #else

-    int mode = BUFF_MODE;
-    size_t fd = 0; 
-    fd = hos_open_fd(handle, bucket, object, callback, (void *)&fd, 0, mode);
+    mode = BUFF_MODE;
+    fd = 0; 
+    fd = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
    debuginfo("hos_upload_buf start ...\n");
    clock_gettime(CLOCK_MONOTONIC, &start);
    for (i = 0; i < test_times; i++)
@@ -135,7 +137,13 @@ int main(int argc, char *argv[])

 #endif
    debuginfo("hos_client_close start ...\n");
-    hos_client_destory(handle);
+    if (hos_client_destory(handle) == 0)
+    {
+        time = calc_time(start, finished);
+        time /= test_times;
+        printf("hos upload finished spent %llu ns\n", time);
+    }
+
    debuginfo("hos_client_close end ...\n");

    return 0;
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.5)

-SET(lib_name hos_client_cpp)
+#SET(lib_name hos_client_cpp)
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC -std=c++11")
 SET(CMAKE_BUILD_TYPE Debug)

@@ -10,6 +10,7 @@ add_library(${lib_name}_shared SHARED hos_client.cpp hos_hash.cpp)
 target_link_libraries(${lib_name}_shared libaws-cpp-sdk-s3.so libaws-cpp-sdk-core.so)
 set_target_properties(${lib_name}_shared PROPERTIES OUTPUT_NAME ${lib_name})

-install(TARGETS ${lib_name}_shared LIBRARY DESTINATION /opt/MESA/lib)
+add_dependencies(${lib_name}_shared aws-sdk-cpp-master-static)
+#install(TARGETS ${lib_name}_shared LIBRARY DESTINATION /opt/MESA/lib)

-#install(TARGETS ${lib_name}_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT LIBRARIES)
+install(TARGETS ${lib_name}_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT LIBRARIES)
--- a/src/hos_client.cpp
+++ b/src/hos_client.cpp
@@ -23,7 +23,7 @@ typedef struct hos_client_handle_s
 {
    Aws::S3::S3Client *S3Client;
    size_t append_size;
-    int thread_sum;
+    size_t thread_sum;
    Aws::Vector<Aws::S3::Model::Bucket> buckets;
 }hos_client_handle_t;

@@ -55,7 +55,7 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,
    }
    const Aws::String& uuid = context->GetUUID();
    size_t thread_id, fd;
-    sscanf(uuid.c_str(), "%llu %llu", &thread_id, &fd);
+    sscanf(uuid.c_str(), "%lu %lu", &thread_id, &fd);
    hos_info_t *hos_info = find_info_by_fd(hash_hos_info[thread_id], fd);
    //put_finished_callback& callback = *(put_finished_callback *)hos_info->callback;
    put_finished_callback callback = (put_finished_callback)hos_info->callback;
@@ -64,7 +64,6 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,

 hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum)
 {
-    int i;
    if (!endpoint || !accesskeyid || !secretkey || thread_sum > MAX_THREAD_NUM)
    {
        return NULL;
@@ -193,7 +192,7 @@ static int hos_upload_stream(hos_client_handle handle, const char *bucket, const
    //设置回调函数
    std::shared_ptr<Aws::Client::AsyncCallerContext> context = 
        Aws::MakeShared<Aws::Client::AsyncCallerContext>("");
-    sprintf(buf, "%ld %d", thread_id, fd);
+    sprintf(buf, "%lu %lu", thread_id, fd);
    context->SetUUID(buf);
    
    hos_info_t info = {fd, 0, handle, bucket, object, (void *)callback, userdata, };
@@ -265,10 +264,15 @@ int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id
    if (hos_info->mode & BUFF_MODE)
    {
        //BUFF_MODE
+#if 0
        const std::shared_ptr<Aws::IOStream> input_data = 
            Aws::MakeShared<Aws::StringStream>(stream, stream + stream_len);
        Aws::String buffer (stream, stream_len);
        *input_data << buffer;
+#else
+        Aws::StringStream *buffer = new Aws::StringStream(Aws::String(stream, stream_len));
+        const std::shared_ptr<Aws::IOStream> input_data(buffer); 
+#endif
        request.SetBody(input_data);
    }
    else
@@ -287,7 +291,7 @@ int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id
    //设置回调函数
    std::shared_ptr<Aws::Client::AsyncCallerContext> context = 
        Aws::MakeShared<Aws::Client::AsyncCallerContext>("");
-    sprintf(buf, "%ld %d", thread_id, fd);
+    sprintf(buf, "%lu %lu", thread_id, fd);
    context->SetUUID(buf);
    
    S3Client.PutObjectAsync(request, PutObjectAsyncFinished, context);
@@ -308,7 +312,7 @@ int hos_close_fd(size_t fd, size_t thread_id)

 int hos_client_destory(hos_client_handle handle)
 {
-    int i = 0;
+    size_t i = 0;
    if (handle == NULL)
    {
        return HOS_PARAMETER_ERROR;
--- a/support/CMakeLists.txt
+++ b/support/CMakeLists.txt
@@ -2,24 +2,23 @@ include(ExternalProject)

 set(AWSS3_ROOT            ${CMAKE_CURRENT_BINARY_DIR})
 set(AWSS3_URL             ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-master.zip)
-set(AWSS3_URL_MD5         a8416a80b15f573e7ac790ca354c8c71)
-set(AWSS3_CONFIGURE       cd ${AWSS3_ROOT}/aws-sdk-cpp-master/ && mkdir build)
-set(AWSS3_MAKE            cd ${AWSS3_ROOT}/aws-sdk-cpp-master/build && cmake .. -DBUILD_ONLY="s3" && make) 
-set(AWSS3_INSTALL         cd ${AWSS3_ROOT}/aws-sdk-cpp-master/build && make install PREFIX=${SUPPORT_INSTALL_PREFIX})
+set(AWSS3_URL_MD5         a94cce4fe5003acf55fe1eac8c49ad4f)
+set(AWSS3_CONFIGURE       cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && cmake . -DBUILD_ONLY=s3)
+set(AWSS3_MAKE            cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make) 
+set(AWSS3_INSTALL         cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make install PREFIX=${SUPPORT_INSTALL_PREFIX})

-ExternalProject_Add(luajit2
-        PREFIX              luajit2
+ExternalProject_Add(aws-sdk-cpp-master
+        PREFIX              aws-sdk-cpp-master
        URL                 ${AWSS3_URL}
        URL_MD5             ${AWSS3_URL_MD5}
        CONFIGURE_COMMAND   ${AWSS3_CONFIGURE}
-        CONFIGURE_COMMAND   ""
        BUILD_COMMAND       ${AWSS3_MAKE}
        INSTALL_COMMAND     ${AWSS3_INSTALL}
        BUILD_IN_SOURCE     1
        )


-add_library(luajit2-static STATIC IMPORTED GLOBAL)
-add_dependencies(luajit2-static luajit2)
-set_property(TARGET luajit2-static PROPERTY IMPORTED_LOCATION ${AWSS3_ROOT}/install/lib/libluajit-5.1.a)
+    add_library(aws-sdk-cpp-master-static STATIC IMPORTED GLOBAL)
+    add_dependencies(aws-sdk-cpp-master-static aws-sdk-cpp-master)
+    set_property(TARGET aws-sdk-cpp-master-static PROPERTY IMPORTED_LOCATION ${SUPPORT_INSTALL_PREFIX})

--- a/support/aws-sdk-cpp-master.zip
+++ b/support/aws-sdk-cpp-master.zip