gfwleak/pxz-hos-client-cpp-module
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加测试客户端程序

Browse Source
This commit is contained in:
pengxuanzheng
2021-04-26 13:59:36 +08:00
committed by 彭宣正
parent bba4e436a3
commit 9377d7912c
10 changed files with 213 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab-ci.yml
View File

@@ -3,7 +3,7 @@ variables:
GIT_STRATEGY: "clone"
BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
INSTALL_PREFIX: "/home/mesasoft/sapp_run/"
INSTALL_DEPENDENCY_LIBRARY: framework_env libMESA_field_stat2-devel zlib-devel openssl-devel curl-devel
INSTALL_DEPENDENCY_LIBRARY: framework_env libMESA_field_stat2-devel zlib-devel openssl-devel curl-devel libMESA_prof_load libMESA_handle_logger
TESTING_VERSION_BUILD: 0
stages:

16
example/demo/CMakeLists.txt
View File

@@ -1,16 +1,16 @@
cmake_minimum_required(VERSION 3.5)
set(CMAKE_BUILD_TYPE Debug)
project(hos_write_complete)
project(hos_upload_complete)
project(hos_write_demo)
#project(hos_upload_complete)
SET(CMAKE_BUILD_TYPE Debug)
link_directories(/usr/local/lib64/)
#link_directories(/usr/local/lib64/)
link_directories(/opt/MESA/lib/)
link_libraries(hos-client-cpp)
#link_libraries(hos-client-cpp)
include_directories(/opt/MESA/include)
add_executable(hos_write_complete hos_write_complete.cpp)
add_executable(hos_upload_complete hos_upload_complete.cpp)
target_link_libraries(hos_write_complete hos-client-cpp)
target_link_libraries(hos_upload_complete hos-client-cpp)
add_executable(hos_write_demo hos_write_demo.cpp)
#add_executable(hos_upload_complete hos_upload_complete.cpp)
target_link_libraries(hos_write_demo hos-client-cpp)
#target_link_libraries(hos_upload_complete hos-client-cpp)

16
example/demo/conf/default.conf Normal file
View File

@@ -0,0 +1,16 @@
[hos_default_conf]
hos_serverip=192.168.40.223
hos_serverport=9098
hos_accesskeyid="default"
hos_secretkey="default"
hos_log_path="./log/hoslog" #default
hos_log_level=30 #default
hos_poolsize=0 #default
hos_thread_sum=32 #default
hos_cache_size=102400 #default
hos_cache_count=10 #default
hos_fd_live_time_ms=1000 #default
hos_fs2_serverip=127.0.0.1
hos_fs2_serverport=10086
hos_fs2_path="./log/hos_fs2_log" #default
hos_fs2_format=0 #defaul

155
example/demo/hos_write_complete.cpp
View File

@@ -1,155 +0,0 @@
/*************************************************************************
> File Name: single_thread.cpp
> Author: pxz
> Created Time: Fri 11 Sep 2020 09:52:05 AM CST
************************************************************************/
extern "C"
{
#include<stdio.h>
#include<stdlib.h>
#include<unistd.h>
#include<string.h>
#include<time.h>
}
#include"hos_client.h"
//#define test_times 10000
#define debuginfo printf
typedef struct userdata_s
{
struct timespec *finished;
}userdata_t;
static size_t calc_time(struct timespec start, struct timespec end)
{
return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec -
(start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
}
int file_to_buffer(const char *file, char *buffer, size_t *len)
{
FILE *fp = fopen(file, "r");
int num = 0;
*len = 0;
if (fp == NULL)
{
debuginfo("fopen file failed:%s\n", file);
return -1;
}
do{
num = fread(&buffer[*len], 1, 4096, fp);
if (num < 0)
{
return -1;
}
*len += num;
}while(num == 4096);
fclose(fp);
return 0;
}
void callback(bool result, const char *error, const char *bucket, const char *object, void *userdata)
{
userdata_t *data = (userdata_t *)userdata;
clock_gettime(CLOCK_MONOTONIC, data->finished);
return ;
}
int main(int argc, char *argv[])
{
if (argc != 3)
{
debuginfo("usege: singThread [bucket name] [object name]\n");
return -1;
}
struct timespec start, end, finished;
size_t time;
int i = 0;
char *bucket = argv[1];
char *object = argv[2];
char *buf = (char *)malloc(1024 * 1024 * 40);
size_t buf_size;
int mode = FILE_MODE;
size_t fd[10001] = {0};
userdata_t data = {&finished};
file_to_buffer(object, buf, &buf_size);
debuginfo("hos_client_init start ...\n");
hos_client_handle handle = hos_client_create("192.168.44.12", 9098, "default", "default", 100);
//hos_client_handle handle = hos_client_create("http://192.168.32.10:9098/hos/", "default", "default", 4);
if (handle == NULL)
{
printf("error:hos_client_handle\n");
return -1;
}
debuginfo("hos_client_init success ... \n");
debuginfo("hos_create_bucket start ... \n");
if(hos_create_bucket(handle, bucket))
{
printf("hos_create_bucket failed ... \n");
return -1;
}
debuginfo("hos_create_bucket success ... \n");
debuginfo("hos_verify_bucket start ... \n");
if(!hos_verify_bucket(handle, bucket))
{
printf("hos_verify_bucket failed ... \n");
return -1;
}
debuginfo("hos_verify_bucket success ... \n");
#if 1
mode = FILE_MODE;
debuginfo("hos_upload_file start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start);
fd[0] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
hos_write(fd[0], object, 0, 0);
clock_gettime(CLOCK_MONOTONIC, &end);
time = calc_time(start, end);
printf("hos_upload_file spent %llu ns\n", time);
debuginfo("hos_upload_file end ...\n");
#else
mode = BUFF_MODE | APPEND_MODE;
#if 0
for (i = 0; i < test_times; i++)
{
}
#endif
debuginfo("hos_upload_buf start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < test_times; i++)
{
fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
//printf("hos_write start...\n");
if(hos_write(fd[i], buf, buf_size, 0, i + 1) != HOS_CLIENT_OK)
{
printf("error: hos_write!\n");
}
//printf("hos_write success...\n");
}
clock_gettime(CLOCK_MONOTONIC, &end);
time = calc_time(start, end);
time /= test_times;
printf("hos_upload_buf spent %llu ns\n", time);
debuginfo("hos_upload_buf end ...\n");
#endif
debuginfo("hos_client_close start ...\n");
if (hos_client_destory(handle) == 0)
{
//printf("start:%lu\n", start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec);
//printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec);
time = calc_time(start, finished);
printf("hos upload finished spent %llu ns\n", time);
}
debuginfo("hos_client_close end ...\n");
free(buf);
return 0;
}

155
example/demo/hos_write_demo.cpp Normal file
View File

@@ -0,0 +1,155 @@
/*************************************************************************
> File Name: single_thread.cpp
> Author: pxz
> Created Time: Fri 11 Sep 2020 09:52:05 AM CST
************************************************************************/
extern "C"
{
#include<stdio.h>
#include<stdlib.h>
#include<unistd.h>
#include<string.h>
#include<time.h>
#include<sys/stat.h>
}
#include"hos_client.h"
//#define test_times 10000
typedef struct userdata_s
{
struct timespec *finished;
}userdata_t;
static size_t calc_time(struct timespec start, struct timespec end)
{
return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec -
(start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
}
int file_to_buffer(const char *file, char *buffer, size_t *len)
{
FILE *fp = fopen(file, "r");
int num = 0;
*len = 0;
if (fp == NULL)
{
printf("fopen file failed:%s\n", file);
return -1;
}
do{
num = fread(&buffer[*len], 1, 4096, fp);
if (num < 0)
{
return -1;
}
*len += num;
}while(num == 4096);
fclose(fp);
return 0;
}
void callback(bool result, const char *error, const char *bucket, const char *object, void *userdata)
{
userdata_t *data = (userdata_t *)userdata;
clock_gettime(CLOCK_MONOTONIC, data->finished);
return ;
}
int main(int argc, char *argv[])
{
if (argc != 4)
{
printf("usege: [conf file] [module name] [file name]\n");
return -1;
}
struct timespec start, end, finished;
size_t time;
int i = 0;
char *conf_file = argv[1];
char *module_name = argv[2];
char *file_name = argv[3];
struct stat buffer;
char *buf = NULL;
size_t buf_size;
int mode = FILE_MODE;
size_t fd[10001] = {0};
userdata_t data = {&finished};
hos_instance hos_instance = NULL;
char object[1024];
if (stat(file_name, &buffer) == -1)
{
printf("%s not exits.", conf_file);
return -1;
}
buf = (char *)calloc(1, buffer.st_size);
if (file_to_buffer(file_name, buf, &buf_size) == -1)
{
free(buf);
return -1;
}
printf("hos_init_instance start ...\n");
hos_instance = hos_get_instance();
if (hos_instance->result == false)
{
hos_instance = hos_init_instance(conf_file, module_name, 1, "hos_test_bucket");
}
if (hos_instance->result == false)
{
printf("error:hos_init_instance\n");
printf("error:%s", hos_instance->error_message);
return -1;
}
printf("hos_init_instance success ... \n");
mode = FILE_MODE;
printf("hos_write file start ...\n");
snprintf(object, 1023, "%s_write_file", file_name);
fd[0] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode);
if (hos_write(fd[i], file_name, 0, 0) != HOS_CLIENT_OK)
{
printf("error: hos_write fialed!\n");
}
hos_close_fd(fd[1], 0);
printf("hos_write file end ...\n");
mode = BUFF_MODE;
printf("hos_write buff start ...\n");
snprintf(object, 1023, "%s_write_buff", file_name);
fd[1] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode);
if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK)
{
printf("error: hos_write failed!\n");
}
hos_close_fd(fd[1], 0);
printf("hos_write buff end ...\n");
mode = BUFF_MODE | APPEND_MODE;
printf("hos_write buff start ...\n");
snprintf(object, 1023, "%s_write_APPEND", file_name);
fd[2] = hos_open_fd("hos_test_bucket", object, callback, NULL, 0, mode);
if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK)
{
printf("error: hos_write failed 1st!\n");
}
if (hos_write(fd[i], buf, buffer.st_size, 0) != HOS_CLIENT_OK)
{
printf("error: hos_write failed 2nd!\n");
}
hos_close_fd(fd[2], 0);
printf("hos_write buff end ...\n");
printf("hos_shutdown_instance start ...\n");
if (hos_shutdown_instance() != 0)
{
printf("error: hos_shutdown_instance fialed\n");
}
printf("hos_shutdown_instance end ...\n");
free(buf);
return 0;
}

9
src/CMakeLists.txt
View File

@@ -9,12 +9,17 @@ set(CMKAE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC")
add_library(${lib_name}_shared SHARED hos_client.cpp hos_hash.cpp)
target_link_libraries(${lib_name}_shared
"-Wl,--whole-archive"
libaws-c-common.a
libaws-checksums.a
libaws-c-event-stream.a
libaws-cpp-sdk-core.a
libaws-cpp-sdk-s3.a
"-Wl,--no-whole-archive"
libMESA_field_stat2.so
libcurl.so
libpthread.so)
libpthread.so
libMESA_prof_load.so
libMESA_field_stat2.so
libMESA_handle_logger.so)
set_target_properties(${lib_name}_shared PROPERTIES OUTPUT_NAME ${lib_name})
#for ASAN

35
src/hos_client.cpp
View File

@@ -237,6 +237,13 @@ static void hos_client_create()
hos_config_t *hos_conf = &g_hos_handle.hos_config;
void *log = g_hos_handle.log;
if (g_hos_handle.S3Client != NULL)
{
g_hos_handle.count++;
g_hos_instance.result = true;
return ;
}
Aws::InitAPI(g_options);
Aws::Client::ClientConfiguration config;
Aws::Auth::AWSCredentials credentials(hos_conf->accesskeyid, hos_conf->secretkey);
@@ -626,6 +633,18 @@ static bool hos_putobject_sync(Aws::S3::Model::PutObjectRequest& request, size_t
}
}
hos_instance hos_get_instance()
{
if (g_hos_handle.S3Client != NULL)
{
g_hos_handle.count++;
g_hos_instance.result = true;
return &g_hos_instance;
}
g_hos_instance.result = false;
return &g_hos_instance;
}
hos_instance hos_init_instance(const char *conf_path, const char *module, size_t thread_num, const char *bucket)
{
hos_config_t *hos_conf = &g_hos_handle.hos_config;
@@ -650,12 +669,13 @@ hos_instance hos_init_instance(const char *conf_path, const char *module, size_t
MESA_load_profile_uint_def(conf_path, module, "hos_cache_size", &hos_conf->cache_size, 102400);
MESA_load_profile_uint_def(conf_path, module, "hos_cache_count", &hos_conf->cache_count, 10);
MESA_load_profile_uint_def(conf_path, module, "hos_fd_live_time_ms", &hos_conf->timeout, 1000);
MESA_load_profile_string_nodef(conf_path, module, "hos_fs2_serverip", hos_conf->fs2_ip, MAX_HOS_STRING_LEN);
MESA_load_profile_string_nodef(conf_path, module, "hos_fs2_serverip", hos_conf->fs2_ip, INET6_ADDRSTRLEN);
MESA_load_profile_uint_nodef(conf_path, module, "hos_fs2_serverport", &hos_conf->fs2_port);
MESA_load_profile_string_def(conf_path, module, "hos_fs2_path", hos_conf->fs2_path, sizeof(hos_conf->fs2_path), "./hos_fs2.stat");
MESA_load_profile_uint_def(conf_path, module, "hos_fs2_format", &hos_conf->fs2_fmt, 0);
if (hos_conf->ip && hos_conf->port && hos_conf->accesskeyid && hos_conf->secretkey)
if (hos_conf->ip && hos_conf->port && strlen(hos_conf->accesskeyid) && strlen(hos_conf->secretkey))
{
MESA_handle_runtime_log_creation("./log");
g_hos_handle.log = MESA_create_runtime_log_handle(hos_conf->log_path, hos_conf->log_level);
if (log == NULL)
{
@@ -1092,10 +1112,7 @@ int hos_close_fd(size_t fd, size_t thread_id)
hos_fd_context_t *a_fd_context = NULL;
char num[128];
hos_config_t *hos_conf = &g_hos_handle.hos_config;
<<<<<<< HEAD
size_t upload_len = 0;
=======
>>>>>>> c9c751ab... 增加同步模式
if (fd < 3 || thread_id > hos_conf->thread_num)
{
@@ -1140,19 +1157,11 @@ int hos_close_fd(size_t fd, size_t thread_id)
if (hos_conf->pool_thread_size > 0)
{
<<<<<<< HEAD
hos_putobject_async(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object);
}
else
{
hos_putobject_sync(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object);
=======
hos_putobject_async(request, a_fd_context->cache->gcount(), thread_id, fd, a_fd_context->bucket, a_fd_context->object);
}
else
{
hos_putobject_sync(request, a_fd_context->cache->gcount(), thread_id, fd, a_fd_context->bucket, a_fd_context->object);
>>>>>>> c9c751ab... 增加同步模式
}
}
}

7
src/hos_client.h
View File

@@ -105,11 +105,10 @@ hos_instance hos_client_create(const char *serverip, size_t port, const char *ac
*************************************************************************************/
hos_instance hos_init_instance(const char *conf_path, const char *module, size_t thread_num, const char *bucket);
/*************************************************************************************
* 函数名： hos_create_bucket
* 参数: const char *bucket 桶名称
* 返回值： bool 成功返回true，失败返回false
* 函数名： hos_get_instance
* 返回值： hos_instance 成功，result 为true
*************************************************************************************/
//bool hos_verify_bucket(const char *bucket);
hos_instance hos_get_instance();
/*************************************************************************************
* 函数名： hos_create_bucket
* 参数： const char * bucket 桶名称

2
support/CMakeLists.txt
View File

@@ -2,7 +2,7 @@ include(ExternalProject)
set(AWSS3_ROOT ${CMAKE_CURRENT_BINARY_DIR})
set(AWSS3_URL ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-master.zip)
set(AWSS3_URL_MD5 7b89ae5c7c7c01bfe582ca5fbad728de)
set(AWSS3_URL_MD5 a38984c137d7768dec2a7cde02874c32)
set(AWSS3_CONFIGURE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && cmake . -DBUILD_ONLY=s3)
set(AWSS3_MAKE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make)
set(AWSS3_INSTALL cd ${AWSS3_ROOT}/aws-sdk-cpp-master/src/aws-sdk-cpp-master && make install PREFIX=${SUPPORT_INSTALL_PREFIX})

BIN
support/aws-sdk-cpp-master.zip
View File

Binary file not shown.