gfwleak/pxz-hos-client-cpp-module
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Develop 09.22

Browse Source
This commit is contained in:
pengxuanzheng
2020-09-22 17:22:21 +08:00
parent eb41917cb2
commit 10e17cb043
7 changed files with 58 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
CMakeLists.txt
View File

@@ -4,21 +4,22 @@ set(lib_name hos-client-cpp)
project(${lib_name}) project(${lib_name})
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
include(Version) #include(Version)
set(CMAKE_MACOSX_RPATH 0) set(CMAKE_MACOSX_RPATH 0)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
set(CMAKE_INSTALL_PREFIX /opt/MESA/lib) set(CMAKE_INSTALL_PREFIX /opt/MESA)
set(SUPPORT_INSTALL_PREFIX /usr/local/lib64) set(SUPPORT_INSTALL_PREFIX /usr/local/lib64)
add_subdirectory(support) add_subdirectory(support)
add_subdirectory(src) add_subdirectory(src)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhso-client-cpp.so DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT PROFILE) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhos-client-cpp.so DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT PROFILE)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhos-client-cpp.a DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT PROFILE) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/src/hos_client.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT PROFILE)
#install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhos-client-cpp.a DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT PROFILE)
include(Package) #include(Package)

10
example/CMakeLists.txt
View File

@@ -5,9 +5,9 @@ project(singleThread)
SET(CMAKE_BUILD_TYPE Debug) SET(CMAKE_BUILD_TYPE Debug)
link_directories(/usr/local/lib64/) link_directories(/usr/local/lib64/)
link_directories(/opt/MESA/lib/) link_directories(/opt/MESA/lib/)
link_libraries(hos_client_cpp) link_libraries(hos-client-cpp)
#set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}) include_directories(/opt/MESA/include)
add_executable(singleThread single_thread.cpp)
target_link_libraries(singleThread hos_client_cpp) add_executable(singleThread single_thread.cpp)
target_link_libraries(singleThread hos-client-cpp)
#install(TARGETS singleThread DESTINATION ${CMAKE_CURRENT_SOURCE_DIR})

40
example/single_thread.cpp
View File

@@ -17,6 +17,11 @@ extern "C"
#define debuginfo (void) #define debuginfo (void)
typedef struct userdata_s
{
struct timespec *finished;
}userdata_t;
static size_t calc_time(struct timespec start, struct timespec end) static size_t calc_time(struct timespec start, struct timespec end)
{ {
return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec -
@@ -47,12 +52,8 @@ int file_to_buffer(const char *file, char *buffer, size_t *len)
void callback(bool result, const char *error, void *userdata) void callback(bool result, const char *error, void *userdata)
{ {
//debuginfo("result : %s\n", result ? "true":"false"); userdata_t *data = (userdata_t *)userdata;
if (result) clock_gettime(CLOCK_MONOTONIC, data->finished);
return ;
//debuginfo("error: %s\n", error);
//debuginfo("userdata:%s\n", (char *)userdata);
hos_close_fd(*(int *)userdata, 0);
return ; return ;
} }
@@ -63,7 +64,7 @@ int main(int argc, char *argv[])
debuginfo("usege: singThread [bucket name] [object name]\n"); debuginfo("usege: singThread [bucket name] [object name]\n");
return -1; return -1;
} }
struct timespec start, end; struct timespec start, end, finished;
size_t time; size_t time;
int i = 0; int i = 0;
char *bucket = argv[1]; char *bucket = argv[1];
@@ -73,11 +74,14 @@ int main(int argc, char *argv[])
//char *buf = (char *)malloc(1024 * 1024 * 4); //char *buf = (char *)malloc(1024 * 1024 * 4);
char buf[1024 * 1024 * 4]; char buf[1024 * 1024 * 4];
size_t buf_size; size_t buf_size;
int mode = FILE_MODE;
size_t fd = 0;
userdata_t data = {&finished};
file_to_buffer(object, buf, &buf_size); file_to_buffer(object, buf, &buf_size);
debuginfo("hos_client_init start ...\n"); debuginfo("hos_client_init start ...\n");
hos_client_handle handle = hos_client_create("http://192.168.44.12:9098/hos/", "default", "default", 4); hos_client_handle handle = hos_client_create("http://192.168.44.10:9098/hos/", "default", "default", 4);
if (handle == NULL) if (handle == NULL)
{ {
debuginfo("error:hos_client_handle\n"); debuginfo("error:hos_client_handle\n");
@@ -101,10 +105,8 @@ int main(int argc, char *argv[])
} }
debuginfo("hos_verify_bucket success ... \n"); debuginfo("hos_verify_bucket success ... \n");
#if 1 #if 0
int mode = FILE_MODE; fd = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
size_t fd = 0;
fd = hos_open_fd(handle, bucket, object, callback, (void *)&fd, 0, mode);
debuginfo("hos_upload_file start ...\n"); debuginfo("hos_upload_file start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < test_times; i++) for (i = 0; i < test_times; i++)
@@ -118,9 +120,9 @@ int main(int argc, char *argv[])
debuginfo("hos_upload_file end ...\n"); debuginfo("hos_upload_file end ...\n");
#else #else
int mode = BUFF_MODE; mode = BUFF_MODE;
size_t fd = 0; fd = 0;
fd = hos_open_fd(handle, bucket, object, callback, (void *)&fd, 0, mode); fd = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
debuginfo("hos_upload_buf start ...\n"); debuginfo("hos_upload_buf start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < test_times; i++) for (i = 0; i < test_times; i++)
@@ -135,7 +137,13 @@ int main(int argc, char *argv[])
#endif #endif
debuginfo("hos_client_close start ...\n"); debuginfo("hos_client_close start ...\n");
hos_client_destory(handle); if (hos_client_destory(handle) == 0)
{
time = calc_time(start, finished);
time /= test_times;
printf("hos upload finished spent %llu ns\n", time);
}
debuginfo("hos_client_close end ...\n"); debuginfo("hos_client_close end ...\n");
return 0; return 0;

7
src/CMakeLists.txt
View File

@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.5) cmake_minimum_required(VERSION 3.5)
SET(lib_name hos_client_cpp) #SET(lib_name hos_client_cpp)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC -std=c++11") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC -std=c++11")
SET(CMAKE_BUILD_TYPE Debug) SET(CMAKE_BUILD_TYPE Debug)
@@ -10,6 +10,7 @@ add_library(${lib_name}_shared SHARED hos_client.cpp hos_hash.cpp)
target_link_libraries(${lib_name}_shared libaws-cpp-sdk-s3.so libaws-cpp-sdk-core.so) target_link_libraries(${lib_name}_shared libaws-cpp-sdk-s3.so libaws-cpp-sdk-core.so)
set_target_properties(${lib_name}_shared PROPERTIES OUTPUT_NAME ${lib_name}) set_target_properties(${lib_name}_shared PROPERTIES OUTPUT_NAME ${lib_name})
install(TARGETS ${lib_name}_shared LIBRARY DESTINATION /opt/MESA/lib) add_dependencies(${lib_name}_shared aws-sdk-cpp-master-static)
#install(TARGETS ${lib_name}_shared LIBRARY DESTINATION /opt/MESA/lib)
#install(TARGETS ${lib_name}_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT LIBRARIES) install(TARGETS ${lib_name}_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT LIBRARIES)

16
src/hos_client.cpp
View File

@@ -23,7 +23,7 @@ typedef struct hos_client_handle_s
{ {
Aws::S3::S3Client *S3Client; Aws::S3::S3Client *S3Client;
size_t append_size; size_t append_size;
int thread_sum; size_t thread_sum;
Aws::Vector<Aws::S3::Model::Bucket> buckets; Aws::Vector<Aws::S3::Model::Bucket> buckets;
}hos_client_handle_t; }hos_client_handle_t;
@@ -55,7 +55,7 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,
} }
const Aws::String& uuid = context->GetUUID(); const Aws::String& uuid = context->GetUUID();
size_t thread_id, fd; size_t thread_id, fd;
sscanf(uuid.c_str(), "%llu %llu", &thread_id, &fd); sscanf(uuid.c_str(), "%lu %lu", &thread_id, &fd);
hos_info_t *hos_info = find_info_by_fd(hash_hos_info[thread_id], fd); hos_info_t *hos_info = find_info_by_fd(hash_hos_info[thread_id], fd);
//put_finished_callback& callback = *(put_finished_callback *)hos_info->callback; //put_finished_callback& callback = *(put_finished_callback *)hos_info->callback;
put_finished_callback callback = (put_finished_callback)hos_info->callback; put_finished_callback callback = (put_finished_callback)hos_info->callback;
@@ -64,7 +64,6 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,
hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum) hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum)
{ {
int i;
if (!endpoint || !accesskeyid || !secretkey || thread_sum > MAX_THREAD_NUM) if (!endpoint || !accesskeyid || !secretkey || thread_sum > MAX_THREAD_NUM)
{ {
return NULL; return NULL;
@@ -193,7 +192,7 @@ static int hos_upload_stream(hos_client_handle handle, const char *bucket, const
//设置回调函数 //设置回调函数
std::shared_ptr<Aws::Client::AsyncCallerContext> context = std::shared_ptr<Aws::Client::AsyncCallerContext> context =
Aws::MakeShared<Aws::Client::AsyncCallerContext>(""); Aws::MakeShared<Aws::Client::AsyncCallerContext>("");
sprintf(buf, "%ld %d", thread_id, fd); sprintf(buf, "%lu %lu", thread_id, fd);
context->SetUUID(buf); context->SetUUID(buf);
hos_info_t info = {fd, 0, handle, bucket, object, (void *)callback, userdata, }; hos_info_t info = {fd, 0, handle, bucket, object, (void *)callback, userdata, };
@@ -265,10 +264,15 @@ int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id
if (hos_info->mode & BUFF_MODE) if (hos_info->mode & BUFF_MODE)
{ {
//BUFF_MODE //BUFF_MODE
#if 0
const std::shared_ptr<Aws::IOStream> input_data = const std::shared_ptr<Aws::IOStream> input_data =
Aws::MakeShared<Aws::StringStream>(stream, stream + stream_len); Aws::MakeShared<Aws::StringStream>(stream, stream + stream_len);
Aws::String buffer (stream, stream_len); Aws::String buffer (stream, stream_len);
*input_data << buffer; *input_data << buffer;
#else
Aws::StringStream *buffer = new Aws::StringStream(Aws::String(stream, stream_len));
const std::shared_ptr<Aws::IOStream> input_data(buffer);
#endif
request.SetBody(input_data); request.SetBody(input_data);
} }
else else
@@ -287,7 +291,7 @@ int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id
//设置回调函数 //设置回调函数
std::shared_ptr<Aws::Client::AsyncCallerContext> context = std::shared_ptr<Aws::Client::AsyncCallerContext> context =
Aws::MakeShared<Aws::Client::AsyncCallerContext>(""); Aws::MakeShared<Aws::Client::AsyncCallerContext>("");
sprintf(buf, "%ld %d", thread_id, fd); sprintf(buf, "%lu %lu", thread_id, fd);
context->SetUUID(buf); context->SetUUID(buf);
S3Client.PutObjectAsync(request, PutObjectAsyncFinished, context); S3Client.PutObjectAsync(request, PutObjectAsyncFinished, context);
@@ -308,7 +312,7 @@ int hos_close_fd(size_t fd, size_t thread_id)
int hos_client_destory(hos_client_handle handle) int hos_client_destory(hos_client_handle handle)
{ {
int i = 0; size_t i = 0;
if (handle == NULL) if (handle == NULL)
{ {
return HOS_PARAMETER_ERROR; return HOS_PARAMETER_ERROR;

19
support/CMakeLists.txt
View File

@@ -2,24 +2,23 @@ include(ExternalProject)
set(AWSS3_ROOT ${CMAKE_CURRENT_BINARY_DIR}) set(AWSS3_ROOT ${CMAKE_CURRENT_BINARY_DIR})
set(AWSS3_URL ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-master.zip) set(AWSS3_URL ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-master.zip)
set(AWSS3_URL_MD5 a8416a80b15f573e7ac790ca354c8c71) set(AWSS3_URL_MD5 a94cce4fe5003acf55fe1eac8c49ad4f)
set(AWSS3_CONFIGURE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/ && mkdir build) set(AWSS3_CONFIGURE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && cmake . -DBUILD_ONLY=s3)
set(AWSS3_MAKE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/build && cmake .. -DBUILD_ONLY="s3" && make) set(AWSS3_MAKE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make)
set(AWSS3_INSTALL cd ${AWSS3_ROOT}/aws-sdk-cpp-master/build && make install PREFIX=${SUPPORT_INSTALL_PREFIX}) set(AWSS3_INSTALL cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make install PREFIX=${SUPPORT_INSTALL_PREFIX})
ExternalProject_Add(luajit2 ExternalProject_Add(aws-sdk-cpp-master
PREFIX luajit2 PREFIX aws-sdk-cpp-master
URL ${AWSS3_URL} URL ${AWSS3_URL}
URL_MD5 ${AWSS3_URL_MD5} URL_MD5 ${AWSS3_URL_MD5}
CONFIGURE_COMMAND ${AWSS3_CONFIGURE} CONFIGURE_COMMAND ${AWSS3_CONFIGURE}
CONFIGURE_COMMAND ""
BUILD_COMMAND ${AWSS3_MAKE} BUILD_COMMAND ${AWSS3_MAKE}
INSTALL_COMMAND ${AWSS3_INSTALL} INSTALL_COMMAND ${AWSS3_INSTALL}
BUILD_IN_SOURCE 1 BUILD_IN_SOURCE 1
) )
add_library(luajit2-static STATIC IMPORTED GLOBAL) add_library(aws-sdk-cpp-master-static STATIC IMPORTED GLOBAL)
add_dependencies(luajit2-static luajit2) add_dependencies(aws-sdk-cpp-master-static aws-sdk-cpp-master)
set_property(TARGET luajit2-static PROPERTY IMPORTED_LOCATION ${AWSS3_ROOT}/install/lib/libluajit-5.1.a) set_property(TARGET aws-sdk-cpp-master-static PROPERTY IMPORTED_LOCATION ${SUPPORT_INSTALL_PREFIX})

BIN
support/aws-sdk-cpp-master.zip
View File

Binary file not shown.