gfwleak/pxz-hos-client-cpp-module
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改并增加新接口

Browse Source
This commit is contained in:
pengxuanzheng
2020-09-21 19:19:18 +08:00
parent a00d892928
commit eb41917cb2
22 changed files with 1916 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
CMakeLists.txt Normal file
View File

@@ -0,0 +1,24 @@
cmake_minimum_required (VERSION 2.8)
set(lib_name hos-client-cpp)
project(${lib_name})
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
include(Version)
set(CMAKE_MACOSX_RPATH 0)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
set(CMAKE_INSTALL_PREFIX /opt/MESA/lib)
set(SUPPORT_INSTALL_PREFIX /usr/local/lib64)
add_subdirectory(support)
add_subdirectory(src)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhso-client-cpp.so DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT PROFILE)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/libhos-client-cpp.a DESTINATION ${CMAKE_INSTALL_PREFIX} COMPONENT PROFILE)
include(Package)

BIN
example/data/100k.data Normal file
View File

Binary file not shown.

BIN
example/data/10k.data Normal file
View File

Binary file not shown.

BIN
example/data/1M.data Normal file
View File

Binary file not shown.

BIN
example/data/1k.data Normal file
View File

Binary file not shown.

BIN
example/data/2M.data Normal file
View File

Binary file not shown.

BIN
example/data/3M.data Normal file
View File

Binary file not shown.

BIN
example/data/4M.data Normal file
View File

Binary file not shown.

15
example/data/test_size.sh Executable file
View File

@@ -0,0 +1,15 @@
#########################################################################
# File Name: test_times.sh
# Author: pxz
# Created Time: Mon 21 Sep 2020 04:43:35 PM CST
#########################################################################
#!/bin/bash
test_size=("1k" "10k" "100k" "1M" "2M" "3M" "4M")
num=0
while((${num} < 7))
do
./singleThread mybucket ${test_size[$num]}.data 1000
let "num++"
done

BIN
example/singleThread Executable file
View File

Binary file not shown.

130
example/single_thread.cpp
View File

@@ -3,34 +3,140 @@
> Author: pxz > Author: pxz
> Created Time: Fri 11 Sep 2020 09:52:05 AM CST > Created Time: Fri 11 Sep 2020 09:52:05 AM CST
************************************************************************/ ************************************************************************/
extern "C"
{
#include<stdio.h> #include<stdio.h>
#include<stdlib.h>
#include<unistd.h> #include<unistd.h>
#include<string.h>
#include<time.h>
}
#include"../src/hos_client.h" #include"../src/hos_client.h"
//#define test_times 10000
#define debuginfo (void)
static size_t calc_time(struct timespec start, struct timespec end)
{
return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec -
(start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
}
int file_to_buffer(const char *file, char *buffer, size_t *len)
{
FILE *fp = fopen(file, "r");
int num = 0;
*len = 0;
if (fp == NULL)
{
debuginfo("fopen file failed:%s\n", file);
return -1;
}
do{
num = fread(&buffer[*len], 1, 4096, fp);
if (num < 0)
{
return -1;
}
*len += num;
}while(num == 4096);
fclose(fp);
return 0;
}
void callback(bool result, const char *error, void *userdata)
{
//debuginfo("result : %s\n", result ? "true":"false");
if (result)
return ;
//debuginfo("error: %s\n", error);
//debuginfo("userdata:%s\n", (char *)userdata);
hos_close_fd(*(int *)userdata, 0);
return ;
}
int main(int argc, char *argv[]) int main(int argc, char *argv[])
{ {
if (argc != 4)
{
debuginfo("usege: singThread [bucket name] [object name]\n");
return -1;
}
struct timespec start, end;
size_t time;
int i = 0;
char *bucket = argv[1]; char *bucket = argv[1];
printf("hos_client_init start ...\n"); char *object = argv[2];
hos_client_handle handle = hos_client_init("http://192.168.44.12:9098/hos", "default", "default"); int test_times = atoi(argv[3]);
//int test_times = 10000;
//char *buf = (char *)malloc(1024 * 1024 * 4);
char buf[1024 * 1024 * 4];
size_t buf_size;
file_to_buffer(object, buf, &buf_size);
debuginfo("hos_client_init start ...\n");
hos_client_handle handle = hos_client_create("http://192.168.44.12:9098/hos/", "default", "default", 4);
if (handle == NULL) if (handle == NULL)
{ {
printf("error:hos_client_handle\n"); debuginfo("error:hos_client_handle\n");
return -1; return -1;
} }
printf("hos_client_init success ..."); debuginfo("hos_client_init success ... \n");
printf("hos_create_bucket start ... \n"); debuginfo("hos_create_bucket start ... \n");
if(!hos_create_bucket(handle, bucket)) if(hos_create_bucket(handle, bucket))
{ {
printf("hos_create_bucket failed ... \n"); debuginfo("hos_create_bucket failed ... \n");
return -1;
} }
printf("hos_create_bucket success ... \n"); debuginfo("hos_create_bucket success ... \n");
printf("hos_upload_async start ...\n"); debuginfo("hos_verify_bucket start ... \n");
hos_upload_async(handle, bucket, "my-file.txt"); if(!hos_verify_bucket(handle, bucket))
{
debuginfo("hos_verify_bucket failed ... \n");
return -1;
}
debuginfo("hos_verify_bucket success ... \n");
//sleep(30); #if 1
hos_client_close(handle); int mode = FILE_MODE;
size_t fd = 0;
fd = hos_open_fd(handle, bucket, object, callback, (void *)&fd, 0, mode);
debuginfo("hos_upload_file start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < test_times; i++)
{
hos_write(fd, object, 0, 0);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time = calc_time(start, end);
time /= test_times;
printf("hos_upload_file spent %llu ns\n", time);
debuginfo("hos_upload_file end ...\n");
#else
int mode = BUFF_MODE;
size_t fd = 0;
fd = hos_open_fd(handle, bucket, object, callback, (void *)&fd, 0, mode);
debuginfo("hos_upload_buf start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < test_times; i++)
{
hos_write(fd, buf, buf_size, 0);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time = calc_time(start, end);
time /= test_times;
printf("hos_upload_buf spent %llu ns\n", time);
debuginfo("hos_upload_buf end ...\n");
#endif
debuginfo("hos_client_close start ...\n");
hos_client_destory(handle);
debuginfo("hos_client_close end ...\n");
return 0; return 0;
} }

115
example/single_thread.cpp_old Normal file
View File

@@ -0,0 +1,115 @@
/*************************************************************************
> File Name: single_thread.cpp
> Author: pxz
> Created Time: Fri 11 Sep 2020 09:52:05 AM CST
************************************************************************/
extern "C"
{
#include<stdio.h>
#include<unistd.h>
#include<string.h>
#include<time.h>
}
#include"../src/hos_client.h"
#define MAX_TEST_TIMES 10
static size_t calc_time(struct timespec start, struct timespec end)
{
return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec -
(start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
}
void callback(bool result, const char *error, void *userdata)
{
//printf("result : %s\n", result ? "true":"false");
if (result)
return ;
//printf("error: %s\n", error);
//printf("userdata:%s\n", (char *)userdata);
return ;
}
int main(int argc, char *argv[])
{
if (argc != 3)
{
printf("usege: singThread [bucket name] [object name]\n");
return -1;
}
struct timespec start, end;
size_t time;
int i = 0;
char *bucket = argv[1];
char *object = argv[2];
printf("hos_client_init start ...\n");
hos_client_handle handle = hos_client_create("http://192.168.44.12:9098/hos/", "default", "default", 4);
if (handle == NULL)
{
printf("error:hos_client_handle\n");
return -1;
}
printf("hos_client_init success ... \n");
printf("hos_create_bucket start ... \n");
if(hos_create_bucket(handle, bucket))
{
printf("hos_create_bucket failed ... \n");
return -1;
}
printf("hos_create_bucket success ... \n");
printf("hos_verify_bucket start ... \n");
if(!hos_verify_bucket(handle, bucket))
{
printf("hos_verify_bucket failed ... \n");
return -1;
}
printf("hos_verify_bucket success ... \n");
printf("hos_upload_file start ...\n");
#if 1
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < MAX_TEST_TIMES; i++)
{
hos_upload_file(handle, bucket, object, callback, (void *)"this is userdata", 0);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time = calc_time(start, end);
time /= MAX_TEST_TIMES;
printf("hos_upload_file spent %llu ns\n", time);
#if 0
for (i = 0; i < MAX_TEST_TIMES; i++)
{
hos_close_fd(i, 0);
}
#endif
printf("hos_upload_file end ...\n");
#else
const char *buf = "this is hos_upload_buf\n";
size_t buf_len = strlen(buf);
printf("hos_upload_buf start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 1; i <= MAX_TEST_TIMES; i++)
{
hos_upload_buf(handle, bucket, object, buf, buf_len, callback, (void *)"this is userdata", 0);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time = calc_time(start, end);
time /= MAX_TEST_TIMES;
printf("hos_upload_buf spent %llu ns\n", time);
for (i = 1; i <= MAX_TEST_TIMES; i++)
{
hos_close_fd(i, 0);
}
printf("hos_upload_buf end ...\n");
#endif
printf("hos_client_close start ...\n");
hos_client_destory(handle);
printf("hos_client_close end ...\n");
return 0;
}

15
example/test_size.sh Executable file
View File

@@ -0,0 +1,15 @@
#########################################################################
# File Name: test_times.sh
# Author: pxz
# Created Time: Mon 21 Sep 2020 04:43:35 PM CST
#########################################################################
#!/bin/bash
test_size=("1k" "10k" "100k" "1M" "2M" "3M" "4M")
num=0
while((${num} < 7))
do
./singleThread mybucket ./data//${test_size[$num]}.data 1000
let "num++"
done

15
example/test_times.sh Executable file
View File

@@ -0,0 +1,15 @@
#########################################################################
# File Name: test_times.sh
# Author: pxz
# Created Time: Mon 21 Sep 2020 04:43:35 PM CST
#########################################################################
#!/bin/bash
test_times=(1, 10, 100, 1000, 10000)
num=0
while((${num} < 5))
do
./singleThread mybucket my-file.txt ${test_times[$num]}
let "num++"
done

4
src/CMakeLists.txt
View File

@@ -4,9 +4,9 @@ SET(lib_name hos_client_cpp)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC -std=c++11") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC -std=c++11")
SET(CMAKE_BUILD_TYPE Debug) SET(CMAKE_BUILD_TYPE Debug)
link_directories(/usr/local/lib64) link_directories(${SUPPORT_INSTALL_PREFIX})
set(CMKAE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC") set(CMKAE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared -fPIC")
add_library(${lib_name}_shared SHARED hos_client.cpp) add_library(${lib_name}_shared SHARED hos_client.cpp hos_hash.cpp)
target_link_libraries(${lib_name}_shared libaws-cpp-sdk-s3.so libaws-cpp-sdk-core.so) target_link_libraries(${lib_name}_shared libaws-cpp-sdk-s3.so libaws-cpp-sdk-core.so)
set_target_properties(${lib_name}_shared PROPERTIES OUTPUT_NAME ${lib_name}) set_target_properties(${lib_name}_shared PROPERTIES OUTPUT_NAME ${lib_name})

304
src/hos_client.cpp
View File

@@ -3,6 +3,10 @@
> Author: pxz > Author: pxz
> Created Time: Thu 10 Sep 2020 03:00:23 PM CST > Created Time: Thu 10 Sep 2020 03:00:23 PM CST
************************************************************************/ ************************************************************************/
extern "C"
{
#include<string.h>
}
#include <aws/core/Aws.h> #include <aws/core/Aws.h>
#include <aws/s3/S3Client.h> #include <aws/s3/S3Client.h>
#include <aws/s3/model/PutObjectRequest.h> #include <aws/s3/model/PutObjectRequest.h>
@@ -13,111 +17,311 @@
#include <mutex> #include <mutex>
#include <sys/stat.h> #include <sys/stat.h>
#include "hos_client.h" #include "hos_client.h"
#include "hos_hash.h"
std::mutex upload_mutex; typedef struct hos_client_handle_s
{
Aws::S3::S3Client *S3Client;
size_t append_size;
int thread_sum;
Aws::Vector<Aws::S3::Model::Bucket> buckets;
}hos_client_handle_t;
static void PutObjectAsyncFinished(const Aws::S3::S3Client* s3Client, #define MAX_THREAD_NUM 255
#define MAX_FD_NUM 65535
hos_info_t *hash_hos_info[MAX_THREAD_NUM];
static size_t hash_get_min_free_fd(hos_info_t *handle)
{
size_t i = 0;
for (i = 1; i < MAX_FD_NUM; i++)
{
if (!find_info_by_fd(handle, i))
return i;
}
return 0;
}
static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,
const Aws::S3::Model::PutObjectRequest& request, const Aws::S3::Model::PutObjectRequest& request,
const Aws::S3::Model::PutObjectOutcome& outcome, const Aws::S3::Model::PutObjectOutcome& outcome,
const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context) const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context)
{ {
if (outcome.IsSuccess()) { const char *error = NULL;
std::cout << "Success: PutObjectAsyncFinished: Finished uploading '" bool result = outcome.IsSuccess();
<< context->GetUUID() << "'." << std::endl; if (!result)
{
error = outcome.GetError().GetMessage().c_str();
} }
else { const Aws::String& uuid = context->GetUUID();
std::cout << "Error: PutObjectAsyncFinished: " << size_t thread_id, fd;
outcome.GetError() << std::endl; sscanf(uuid.c_str(), "%llu %llu", &thread_id, &fd);
} hos_info_t *hos_info = find_info_by_fd(hash_hos_info[thread_id], fd);
//put_finished_callback& callback = *(put_finished_callback *)hos_info->callback;
put_finished_callback callback = (put_finished_callback)hos_info->callback;
callback(result, error, hos_info->userdata);
} }
hos_client_handle hos_client_init(const char *endpoint, const char *accesskeyid, const char *secretkey) hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum)
{ {
if (!endpoint || !accesskeyid || !secretkey) int i;
if (!endpoint || !accesskeyid || !secretkey || thread_sum > MAX_THREAD_NUM)
{ {
return NULL; return NULL;
} }
Aws::SDKOptions options; Aws::SDKOptions options;
Aws::InitAPI(options); Aws::InitAPI(options);
hos_client_handle handle = NULL; hos_client_handle handle = (hos_client_handle)malloc(sizeof(hos_client_handle_t));
memset(handle, 0, sizeof(hos_client_handle_t));
Aws::Client::ClientConfiguration config; Aws::Client::ClientConfiguration config;
Aws::Auth::AWSCredentials credentials(accesskeyid, secretkey); Aws::Auth::AWSCredentials credentials(accesskeyid, secretkey);
//std::cout << "accesskeyid: " << credentials.GetAWSAccessKeyId() << "\n" << std::endl;
//std::cout << "secretkey: " << credentials.GetAWSSecretKey() << "\n" << std::endl;
config.endpointOverride = endpoint; config.endpointOverride = endpoint;
config.verifySSL = false; config.verifySSL = false;
config.enableEndpointDiscovery = true; config.enableEndpointDiscovery = true;
handle = new Aws::S3::S3Client(credentials, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false); handle->S3Client = new Aws::S3::S3Client(credentials, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false);
handle->append_size = 30 * 1024 * 1024;
handle->thread_sum = thread_sum;
/* 获取当前用户的所有的buckets */
Aws::S3::Model::ListBucketsOutcome outcome = handle->S3Client->ListBuckets();
if (outcome.IsSuccess())
{
handle->buckets = outcome.GetResult().GetBuckets();
}
return handle; return handle;
} }
bool hos_verify_bucket(hos_client_handle handle, const char *bucket)
{
Aws::S3::Model::ListBucketsOutcome outcome = handle->S3Client->ListBuckets();
if (outcome.IsSuccess())
{
handle->buckets = outcome.GetResult().GetBuckets();
for (Aws::S3::Model::Bucket& new_bucket : handle->buckets)
{
if (strcmp(new_bucket.GetName().c_str(), bucket) == 0)
{
return true;
}
}
}
return false;
}
int hos_create_bucket(hos_client_handle handle, const char *bucket) int hos_create_bucket(hos_client_handle handle, const char *bucket)
{ {
if (!bucket) if ((bucket == NULL) || (handle == NULL))
{ {
return -1; return HOS_PARAMETER_ERROR;
} }
Aws::S3::S3Client& s3Client = *(Aws::S3::S3Client *) handle; Aws::S3::S3Client& S3Client = *handle->S3Client;
/* 本地检查是否已经存在该bucket */
for (Aws::S3::Model::Bucket& new_bucket : handle->buckets)
{
if (strcmp(new_bucket.GetName().c_str(), bucket) == 0)
{
return HOS_CLIENT_OK;
}
}
Aws::S3::Model::CreateBucketRequest createBucketRequest; Aws::S3::Model::CreateBucketRequest createBucketRequest;
createBucketRequest.SetBucket(bucket); createBucketRequest.SetBucket(bucket);
//std::cout << "bucket name: " << createBucketRequest.GetBucket() << "\n" << std::endl;
Aws::S3::Model::CreateBucketOutcome createBucketOutcome = s3Client.CreateBucket(createBucketRequest); Aws::S3::Model::CreateBucketOutcome createBucketOutcome = S3Client.CreateBucket(createBucketRequest);
if (!createBucketOutcome.IsSuccess()) if (!createBucketOutcome.IsSuccess())
{ {
Aws::S3::S3Errors errorcode = createBucketOutcome.GetError().GetErrorType(); Aws::S3::S3Errors errorcode = createBucketOutcome.GetError().GetErrorType();
if (errorcode != Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) if (errorcode != Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU)
{ {
//std::cout << "Failed to create bucket: " << bucket << "\n" << createBucketOutcome.GetError() << std::endl; return (int)errorcode + 1;
return errorcode;
} }
} }
return 0; //handle->buckets.push_back();
return HOS_CLIENT_OK;
} }
bool hos_upload_async(hos_client_handle handle, const char *bucket, const char *object) static int hos_upload_stream(hos_client_handle handle, const char *bucket, const char *object,
const char *data, size_t data_len, put_finished_callback callback, void *userdata, size_t thread_id, int file_type)
{ {
Aws::S3::S3Client& s3Client = *(Aws::S3::S3Client *) handle;
struct stat buffer; struct stat buffer;
char buf[128];
size_t fd = hash_get_min_free_fd(hash_hos_info[thread_id]);
std::unique_lock<std::mutex> lock(upload_mutex); if ((handle == NULL) || (bucket == NULL) || (object == NULL) || (callback == NULL) || (thread_id > handle->thread_sum))
if (stat(object, &buffer) == -1)
{ {
//error: file does not exist. return HOS_PARAMETER_ERROR;
return false;
} }
Aws::S3::S3Client& S3Client = *handle->S3Client;
// Create and configure the asynchronous put object request. // Create and configure the asynchronous put object request.
Aws::S3::Model::PutObjectRequest request; Aws::S3::Model::PutObjectRequest request;
request.SetBucket(bucket); request.SetBucket(bucket);
request.SetKey(object); request.SetKey(object);
const std::shared_ptr<Aws::IOStream> input_data = //设置上传数据类型
Aws::MakeShared<Aws::FStream>("SampleAllocationTag", object, std::ios_base::in | std::ios_base::binary); if (file_type == 0)
request.SetBody(input_data);
std::shared_ptr<Aws::Client::AsyncCallerContext> context =
Aws::MakeShared<Aws::Client::AsyncCallerContext>("PutObjectAllocationTag");
context->SetUUID(object);
s3Client.PutObjectAsync(request, PutObjectAsyncFinished, context);
return true;
}
void hos_client_close(hos_client_handle handle)
{
if (handle == NULL)
{ {
return; if (stat(object, &buffer) == -1)
{
return HOS_FILE_NOT_EXITS;
}
//文件类型
const std::shared_ptr<Aws::IOStream> input_data =
Aws::MakeShared<Aws::FStream>("SampleAllocationTag", object, std::ios_base::in | std::ios_base::binary);
request.SetBody(input_data);
}
else
{
//内存块
const std::shared_ptr<Aws::IOStream> input_data =
Aws::MakeShared<Aws::StringStream>(data);
Aws::String stream (data, data_len);
*input_data << stream;
request.SetBody(input_data);
} }
delete (Aws::S3::S3Client *)handle; //设置回调函数
std::shared_ptr<Aws::Client::AsyncCallerContext> context =
return ; Aws::MakeShared<Aws::Client::AsyncCallerContext>("");
sprintf(buf, "%ld %d", thread_id, fd);
context->SetUUID(buf);
hos_info_t info = {fd, 0, handle, bucket, object, (void *)callback, userdata, };
add_hos_info(&hash_hos_info[thread_id], &info);
S3Client.PutObjectAsync(request, PutObjectAsyncFinished, context);
return HOS_CLIENT_OK;
}
int hos_upload_file(hos_client_handle handle, const char *bucket, const char *file_path,
put_finished_callback callback, void *userdata, size_t thread_id)
{
return hos_upload_stream(handle, bucket, file_path, NULL, 0, callback, userdata, thread_id, 0);
}
int hos_upload_buf(hos_client_handle handle, const char *bucket, const char *object,
const char *buf, size_t buf_len, put_finished_callback callback, void *userdata, size_t thread_id)
{
return hos_upload_stream(handle, bucket, object, buf, buf_len, callback, userdata, thread_id, 1);
}
int hos_open_fd(hos_client_handle handle, const char *bucket, const char *object, put_finished_callback callback, void *userdata, size_t thread_id, int mode)
{
if ((handle == NULL) || (bucket == NULL) || (object == NULL) || (thread_id > handle->thread_sum))
{
return HOS_PARAMETER_ERROR;
}
size_t fd = hash_get_min_free_fd(hash_hos_info[thread_id]);
if (fd == 0)
{
return HOS_FD_NOT_ENOUGH;
}
hos_info_t info = {fd, mode, handle, bucket, object, (void *)callback, userdata, };
add_hos_info(&hash_hos_info[thread_id], &info);
return fd;
}
int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id)
{
struct stat buffer;
hos_info_t *hos_info = NULL;
hos_client_handle handle = NULL;
char buf[128];
if ((fd == 0) || (stream == NULL) || (thread_id > MAX_THREAD_NUM))
{
return HOS_PARAMETER_ERROR;
}
hos_info = find_info_by_fd(hash_hos_info[thread_id], fd);
if (hos_info == NULL)
{
return HOS_HASH_NOT_FIND;
}
handle = (hos_client_handle)hos_info->handle;
Aws::S3::S3Client& S3Client = *(handle->S3Client);
// Create and configure the asynchronous put object request.
Aws::S3::Model::PutObjectRequest request;
request.SetBucket(hos_info->bucket);
request.SetKey(hos_info->object);
//TODO APPEND MODE
//设置上传数据类型
if (hos_info->mode & BUFF_MODE)
{
//BUFF_MODE
const std::shared_ptr<Aws::IOStream> input_data =
Aws::MakeShared<Aws::StringStream>(stream, stream + stream_len);
Aws::String buffer (stream, stream_len);
*input_data << buffer;
request.SetBody(input_data);
}
else
{
//BUFF_MODE
if (stat(hos_info->object, &buffer) == -1)
{
return HOS_FILE_NOT_EXITS;
}
//文件类型
const std::shared_ptr<Aws::IOStream> input_data =
Aws::MakeShared<Aws::FStream>("SampleAllocationTag", hos_info->object, std::ios_base::in | std::ios_base::binary);
request.SetBody(input_data);
}
//设置回调函数
std::shared_ptr<Aws::Client::AsyncCallerContext> context =
Aws::MakeShared<Aws::Client::AsyncCallerContext>("");
sprintf(buf, "%ld %d", thread_id, fd);
context->SetUUID(buf);
S3Client.PutObjectAsync(request, PutObjectAsyncFinished, context);
return HOS_CLIENT_OK;
}
int hos_close_fd(size_t fd, size_t thread_id)
{
if (fd == 0)
{
return HOS_PARAMETER_ERROR;
}
delete_info_by_fd(hash_hos_info[thread_id], fd);
return HOS_CLIENT_OK;
}
int hos_client_destory(hos_client_handle handle)
{
int i = 0;
if (handle == NULL)
{
return HOS_PARAMETER_ERROR;
}
delete handle->S3Client;
for (i = 0; i < handle->thread_sum; i++)
{
delete_all(hash_hos_info[i]);
}
free(handle);
return HOS_CLIENT_OK;
} }

146
src/hos_client.h
View File

@@ -7,48 +7,62 @@
#define __HOS_CLIENT_INIT__ #define __HOS_CLIENT_INIT__
/*hos client 句柄*/ /*hos client 句柄*/
typedef void* hos_client_handle; typedef struct hos_client_handle_s* hos_client_handle;
#define HOS_CLINET_OK 0 #define HOS_CLIENT_OK 0
#define HOS_CLIENT_ERR -1
/* fd mode */
#define FILE_MODE 0x00
#define BUFF_MODE 0x01
#define APPEND_MODE 0x02 /* 默认不追加 */
/* hos 错误码 */
enum hoserrors
{
HOS_PARAMETER_ERROR = -1,
HOS_FILE_NOT_EXITS = -2,
HOS_HASH_NOT_FIND = -3,
HOS_FD_NOT_ENOUGH = -4,
};
/* s3 的错误码 */ /* s3 的错误码 */
enum s3errors enum s3errors
{ {
//From Core// //From Core Aws::S3::S3Error + 1//
////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////
INCOMPLETE_SIGNATURE = 0, INCOMPLETE_SIGNATURE = 1,
INTERNAL_FAILURE = 1, INTERNAL_FAILURE = 2,
INVALID_ACTION = 2, INVALID_ACTION = 3,
INVALID_CLIENT_TOKEN_ID = 3, INVALID_CLIENT_TOKEN_ID = 4,
INVALID_PARAMETER_COMBINATION = 4, INVALID_PARAMETER_COMBINATION = 5,
INVALID_QUERY_PARAMETER = 5, INVALID_QUERY_PARAMETER = 6,
INVALID_PARAMETER_VALUE = 6, INVALID_PARAMETER_VALUE = 7,
MISSING_ACTION = 7, // SDK should never allow MISSING_ACTION = 8, // SDK should never allow
MISSING_AUTHENTICATION_TOKEN = 8, // SDK should never allow MISSING_AUTHENTICATION_TOKEN = 9, // SDK should never allow
MISSING_PARAMETER = 9, // SDK should never allow MISSING_PARAMETER = 10, // SDK should never allow
OPT_IN_REQUIRED = 10, OPT_IN_REQUIRED = 11,
REQUEST_EXPIRED = 11, REQUEST_EXPIRED = 12,
SERVICE_UNAVAILABLE = 12, SERVICE_UNAVAILABLE = 13,
THROTTLING = 13, THROTTLING = 14,
VALIDATION = 14, VALIDATION = 15,
ACCESS_DENIED = 15, ACCESS_DENIED = 16,
RESOURCE_NOT_FOUND = 16, // Shared with multiple services RESOURCE_NOT_FOUND = 17, // Shared with multiple services
UNRECOGNIZED_CLIENT = 17, // Most likely caused by an invalid access key or secret key UNRECOGNIZED_CLIENT = 18, // Most likely caused by an invalid access key or secret key
MALFORMED_QUERY_STRING = 18, MALFORMED_QUERY_STRING = 19,
SLOW_DOWN = 19, SLOW_DOWN = 20,
REQUEST_TIME_TOO_SKEWED = 20, REQUEST_TIME_TOO_SKEWED = 21,
INVALID_SIGNATURE = 21, INVALID_SIGNATURE = 22,
SIGNATURE_DOES_NOT_MATCH = 22, SIGNATURE_DOES_NOT_MATCH = 23,
INVALID_ACCESS_KEY_ID = 23, INVALID_ACCESS_KEY_ID = 24,
REQUEST_TIMEOUT = 24, REQUEST_TIMEOUT = 25,
NETWORK_CONNECTION = 99, // General failure to send message to service NETWORK_CONNECTION = 100, // General failure to send message to service
// These are needed for logical reasons // These are needed for logical reasons
UNKNOWN = 100, UNKNOWN = 101,
CLIENT_SIGNING_FAILURE = 101, // Client failed to sign the request CLIENT_SIGNING_FAILURE = 102, // Client failed to sign the request
USER_CANCELLED = 102, // User cancelled the request USER_CANCELLED = 103, // User cancelled the request
SERVICE_EXTENSION_START_RANGE = 128 SERVICE_EXTENSION_START_RANGE = 129,
BUCKET_ALREADY_EXISTS= SERVICE_EXTENSION_START_RANGE + 1, BUCKET_ALREADY_EXISTS= SERVICE_EXTENSION_START_RANGE + 1,
BUCKET_ALREADY_OWNED_BY_YOU, BUCKET_ALREADY_OWNED_BY_YOU,
NO_SUCH_BUCKET, NO_SUCH_BUCKET,
@@ -58,32 +72,80 @@ enum s3errors
OBJECT_NOT_IN_ACTIVE_TIER OBJECT_NOT_IN_ACTIVE_TIER
}; };
typedef void (*put_finished_callback)(bool, const char *, void *);
/************************************************************************************* /*************************************************************************************
* 函数名： hos_client_init * 函数名： hos_client_init
* 参数： const char *endpoint 目的地址，如”http://192.168.44.12:9098/hos“ * 参数： const char *endpoint 目的地址，如”http://192.168.44.12:9098/hos“
* const char *accesskeyid AWS access key ID，如”default“ * const char *accesskeyid AWS access key ID，如”default“
* const char *secretkey AWS secret key，如”secretkey“ * const char *secretkey AWS secret key，如”default“
* 返回值： 成功返回一个非空句柄，失败返回NULL。（失败原因都是因为输入参数为空） * 返回值： 成功返回一个非空句柄，失败返回NULL。（失败原因都是因为输入参数为空）
*************************************************************************************/ *************************************************************************************/
hos_client_handle hos_client_init(const char *endpoint, const char *accesskeyid, const char *secretkey); hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_id);
/************************************************************************************* /*************************************************************************************
* 函数名： hos_create_bucket * 函数名： hos_create_bucket
* 参数： hos_client_handle handle 非空句柄 * 参数： hos_client_handle handle 非空句柄
* const char * bucket 桶名称 * const char * bucket 桶名称
* 返回值： bool * 返回值： int 成功返回0，S3错误返回s3errors错误码，hos client错误返回hoserrors错误码
*************************************************************************************/ *************************************************************************************/
bool hos_create_bucket(hos_client_handle handle, const char *bucket); bool hos_verify_bucket(hos_client_handle handle, const char *bucket);
/*************************************************************************************
* 函数名： hos_create_bucket
* 参数： hos_client_handle handle 非空句柄
* const char * bucket 桶名称
* 返回值： int 成功返回0，S3错误返回s3errors错误码，hos client错误返回hoserrors错误码
*************************************************************************************/
int hos_create_bucket(hos_client_handle handle, const char *bucket);
/************************************************************************************* /*************************************************************************************
* 函数名： hos_upload_async * 函数名： hos_upload_async
* 参数： hos_client_handle handle 非空句柄 * 参数： hos_client_handle handle 非空句柄
* const char * bucket 桶名称 * const char * bucket 桶名称
* const char * object 上传对象名称 * const char * object 上传对象名称
* 返回值 bool * put_finished_callback callback upload操作结束时调用的回调函数
* 返回值 int 成功返回0，失败返回hoserros错误码
*************************************************************************************/ *************************************************************************************/
bool hos_upload_async(hos_client_handle handle, const char *bucket, const char *object); int hos_upload_file(hos_client_handle handle, const char *bucket, const char *file_path, put_finished_callback callback, void* userdata, size_t thread_id);
/************************************************************************************* /*************************************************************************************
* 函数名： hos_client_close * 函数名： hos_upload_async
* 参数： hos_client_handle handle 非空句柄 * 参数： hos_client_handle handle 非空句柄
* const char * bucket 桶名称
* const char * object 上传对象名称
* put_finished_callback callback upload操作结束时调用的回调函数
* 返回值 int 成功返回0，失败返回hoserros错误码
*************************************************************************************/ *************************************************************************************/
void hos_client_close(hos_client_handle handle); int hos_upload_buf(hos_client_handle handle, const char *bucket, const char *object, const char *buf, size_t buf_len, put_finished_callback callback, void *userdata, size_t thread_id);
/*************************************************************************************
* 函数名： hos_open_fd
* 参数： hos_client_handle handle 非空句柄
* const char * bucket 桶名称
* const char * object 上传对象名称
* put_finished_callback callback upload操作结束时调用的回调函数
* void *data 用户自定义数据
* size_t thread_id 线程ID
* int mode 模式 (FILE OR BUFFER, APPEND OR NOT)
* 返回值 int 成功返回0，失败返回hoserros错误码
*************************************************************************************/
int hos_open_fd(hos_client_handle handle, const char *bucket, const char *object, put_finished_callback callback, void *userdata, size_t thread_id, int mode);
/*************************************************************************************
* 函数名： hos_upload_stream_async
* 参数： hos_client_handle handle 非空句柄
* const char * stream 待上传的数据
* size_t stream 待上传的数据长度
* size_t thread_id 线程ID
* 返回值 int 成功返回0，失败返回hoserros错误码
*************************************************************************************/
int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id);
/*************************************************************************************
* 函数名： hos_close_fd
* 参数： size_t fd fd
* size_t thread_id 线程ID
* 返回值 int 成功返回0，失败返回hoserros错误码
*************************************************************************************/
int hos_close_fd(size_t fd, size_t thread_id);
/*************************************************************************************
* 函数名： hos_client_destory
* 参数： hos_client_handle handle 非空句柄
* 返回值 int 成功返回0，失败返回hoserros错误码
*************************************************************************************/
int hos_client_destory(hos_client_handle handle);
#endif #endif

52
src/hos_hash.cpp Normal file
View File

@@ -0,0 +1,52 @@
/*************************************************************************
> File Name: uthash.cpp
> Author: pxz
> Created Time: Fri 18 Sep 2020 04:26:09 PM CST
************************************************************************/
#include "hos_hash.h"
void add_hos_info(hos_info_t **handle, hos_info_t *input)
{
hos_info_t *value = NULL;
HASH_FIND_INT(*handle, &input->fd, value);
if (value == NULL)
{
value = (hos_info_t *)malloc(sizeof(hos_info_t));
memcpy(value, input, sizeof(hos_info_t));
HASH_ADD_INT(*handle, fd, value);
}
else
{
value->bucket = input->bucket;
value->object = input->object;
value->callback = input->callback;
value->userdata = input->userdata;
}
}
hos_info_t *find_info_by_fd(hos_info_t *handle, size_t fd)
{
hos_info_t *value = NULL;
HASH_FIND_INT(handle, &fd, value);
return value;
}
void delete_info_by_fd(hos_info_t *handle, size_t fd)
{
hos_info_t *value = NULL;
HASH_FIND_INT(handle, &fd, value);
if (value)
{
HASH_DEL(handle, value);
free(value);
}
}
void delete_all(hos_info_t *handle)
{
hos_info_t *current, *tmp;
HASH_ITER(hh, handle, current, tmp)
{
HASH_DEL(handle, current);
}
}

27
src/hos_hash.h Normal file
View File

@@ -0,0 +1,27 @@
/*************************************************************************
> File Name: hos_hash.h
> Author: pxz
> Created Time: Fri 18 Sep 2020 05:00:04 PM CST
************************************************************************/
#ifndef __HOS_HASH_H__
#define __HOS_HASH_H__
#include "uthash.h"
typedef struct hos_info_s
{
size_t fd;
int mode;
void *handle;
const char *bucket;
const char *object;
void *callback;
void *userdata;
UT_hash_handle hh;
}hos_info_t;
void add_hos_info(hos_info_t **handle, hos_info_t *input);
hos_info_t *find_info_by_fd(hos_info_t *handle, size_t fd);
void delete_info_by_fd(hos_info_t *handle, size_t fd);
void delete_all(hos_info_t *handle);
#endif

1150
src/uthash.h Normal file
View File

File diff suppressed because it is too large Load Diff

25
support/CMakeLists.txt Normal file
View File

@@ -0,0 +1,25 @@
include(ExternalProject)
set(AWSS3_ROOT ${CMAKE_CURRENT_BINARY_DIR})
set(AWSS3_URL ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-master.zip)
set(AWSS3_URL_MD5 a8416a80b15f573e7ac790ca354c8c71)
set(AWSS3_CONFIGURE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/ && mkdir build)
set(AWSS3_MAKE cd ${AWSS3_ROOT}/aws-sdk-cpp-master/build && cmake .. -DBUILD_ONLY="s3" && make)
set(AWSS3_INSTALL cd ${AWSS3_ROOT}/aws-sdk-cpp-master/build && make install PREFIX=${SUPPORT_INSTALL_PREFIX})
ExternalProject_Add(luajit2
PREFIX luajit2
URL ${AWSS3_URL}
URL_MD5 ${AWSS3_URL_MD5}
CONFIGURE_COMMAND ${AWSS3_CONFIGURE}
CONFIGURE_COMMAND ""
BUILD_COMMAND ${AWSS3_MAKE}
INSTALL_COMMAND ${AWSS3_INSTALL}
BUILD_IN_SOURCE 1
)
add_library(luajit2-static STATIC IMPORTED GLOBAL)
add_dependencies(luajit2-static luajit2)
set_property(TARGET luajit2-static PROPERTY IMPORTED_LOCATION ${AWSS3_ROOT}/install/lib/libluajit-5.1.a)

BIN
support/aws-sdk-cpp-master.zip
View File

Binary file not shown.