gfwleak/pxz-hos-client-cpp-module
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🦄 refactor(TSG-7851): 重构fd，使用内存地址作为fd

Browse Source
This commit is contained in:
“pengxuanzheng”
2021-09-24 10:44:30 +08:00
committed by pengxuanzheng
parent 9250031fac
commit f9bce9d590
23 changed files with 138 additions and 1432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
example/demo/conf/default.conf
View File

@@ -1,6 +1,6 @@
[hos_default_conf]
hos_serverip=192.168.40.223
hos_serverport=9098
hos_serverport=9099
hos_accesskeyid="default"
hos_secretkey="default"
hos_log_path="./log/hoslog" #default

1
gtest/CheckHosClient.cpp
View File

@@ -98,7 +98,6 @@ void CheckStructGHosFdContext(hos_fd_context_t *actual, hos_fd_context_t *expect
EXPECT_EQ(actual->cache_count, expect->cache_count);
EXPECT_EQ(actual->cache_rest, expect->cache_rest);
EXPECT_EQ(actual->callback, expect->callback);
EXPECT_EQ(actual->fd, expect->fd);
EXPECT_EQ(actual->fd_status, expect->fd_status);
EXPECT_EQ(actual->mode, expect->mode);
EXPECT_STREQ(actual->object, expect->object);

2
gtest/common/gtest_hos_common.h
View File

@@ -61,7 +61,5 @@ typedef struct hos_fd_context_s hos_fd_context_t;
extern struct hos_instance_s g_hos_instance;
extern hos_client_handle_t g_hos_handle;//一个进程只允许有一个g_hos_handle
extern hos_fd_context_t **g_fd_context;
extern size_t (*g_fd_info)[65536]; //fd 实际从3开始， fd[thread_id][0]记录register的fd，fd[thread_id][1]记录inject的fd
#endif

BIN
gtest/file/test.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test0.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test1.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test2.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test3.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test4.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test5.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test6.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test7.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test8.file Normal file
View File

Binary file not shown.

BIN
gtest/file/test9.file Normal file
View File

Binary file not shown.

10
gtest/gtest_hos_close_fd.cpp
View File

@@ -57,7 +57,6 @@ static void gtest_hos_fd_init(hos_fd_context_t *fd_info)
fd_info->cache_count = 10;
fd_info->cache_rest = g_hos_handle.hos_config.cache_size;
fd_info->callback = NULL;
fd_info->fd = 1;
fd_info->fd_status = HOS_FD_REGISTER;
fd_info->mode = BUFF_MODE | APPEND_MODE;
fd_info->position = 0;
@@ -79,11 +78,10 @@ TEST(hos_close_fd, normal)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
size_t fd = hos_open_fd(HOS_BUCKET, "object", NULL, NULL, 0);
EXPECT_EQ(fd, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info);
CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info);
CheckStructGHosFdContext((hos_fd_context_t *)fd, &expect_fd_info);
int ret = hos_close_fd(fd, 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -107,7 +105,6 @@ TEST(hos_close_fd, normal)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_close_fd, paramer_error)
@@ -128,7 +125,6 @@ TEST(hos_close_fd, paramer_error)
EXPECT_EQ(fd, HOS_PARAMETER_ERROR);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
CheckStructGHosFdContext(g_fd_context[0], NULL);
int ret = hos_close_fd(fd, thread_num + 1);
EXPECT_EQ(ret, HOS_PARAMETER_ERROR);
@@ -152,7 +148,6 @@ TEST(hos_close_fd, paramer_error)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_close_fd, not_init_instance)
@@ -161,6 +156,7 @@ TEST(hos_close_fd, not_init_instance)
EXPECT_EQ(ret, HOS_INSTANCE_NOT_INIT);
}
#if 0
TEST(hos_close_fd, fd_not_exits)
{
hos_instance_s expect_hos_instance;
@@ -196,5 +192,5 @@ TEST(hos_close_fd, fd_not_exits)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
#endif

14
gtest/gtest_hos_open_fd.cpp
View File

@@ -57,7 +57,6 @@ static void gtest_hos_fd_init(hos_fd_context_t *fd_info)
fd_info->cache_count = 10;
fd_info->cache_rest = g_hos_handle.hos_config.cache_size;
fd_info->callback = NULL;
fd_info->fd = 1;
fd_info->fd_status = HOS_FD_REGISTER;
fd_info->mode = BUFF_MODE | APPEND_MODE;
fd_info->position = 0;
@@ -78,30 +77,26 @@ TEST(hos_open_fd, normal)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
size_t fd = hos_open_fd(HOS_BUCKET, "object", NULL, NULL, 0);
EXPECT_EQ(fd, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[0]);
CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info[0]);
CheckStructGHosFdContext((hos_fd_context_t *)fd, &expect_fd_info[0]);
size_t fd1 = hos_open_fd(HOS_BUCKET, "object", NULL, NULL, 1);
EXPECT_EQ(fd1, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[1]);
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
int ret = hos_close_fd(fd, 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
//CheckStructGHosFdContext(g_fd_context[0], NULL);
ret = hos_close_fd(fd1, 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
//CheckStructGHosFdContext(g_fd_context[1], NULL);
ret = hos_shutdown_instance();
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -120,7 +115,6 @@ TEST(hos_open_fd, normal)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_open_fd, paramer_error)
@@ -138,7 +132,6 @@ TEST(hos_open_fd, paramer_error)
EXPECT_EQ(fd, HOS_PARAMETER_ERROR);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
CheckStructGHosFdContext(g_fd_context[0], NULL);
int ret = hos_shutdown_instance();
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -157,7 +150,6 @@ TEST(hos_open_fd, paramer_error)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_open_fd, over_threadnums)
@@ -175,7 +167,6 @@ TEST(hos_open_fd, over_threadnums)
EXPECT_EQ(fd, HOS_PARAMETER_ERROR);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
CheckStructGHosFdContext(g_fd_context[2], NULL);
int ret = hos_shutdown_instance();
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -194,7 +185,6 @@ TEST(hos_open_fd, over_threadnums)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_open_fd, not_init_instance)

115
gtest/gtest_hos_write.cpp
View File

@@ -59,7 +59,6 @@ static void gtest_hos_fd_init(hos_fd_context_t *fd_info)
fd_info->cache_count = 10;
fd_info->cache_rest = g_hos_handle.hos_config.cache_size;
fd_info->callback = NULL;
fd_info->fd = 1;
fd_info->fd_status = HOS_FD_REGISTER;
fd_info->mode = BUFF_MODE | APPEND_MODE;
fd_info->position = 0;
@@ -119,14 +118,13 @@ TEST(hos_write, normal)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
size_t fd = hos_open_fd(HOS_BUCKET, "object_buff", hos_write_buff_cb, (void *)"object_buff", 0);
EXPECT_EQ(fd, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[0]);
expect_fd_info[0].callback = (void *)hos_write_buff_cb;
expect_fd_info[0].object = (char *)"object_buff";
expect_fd_info[0].userdata = (void *)"object_buff";
CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info[0]);
CheckStructGHosFdContext((hos_fd_context_t *)fd, &expect_fd_info[0]);
int ret = hos_write(fd, HOS_BUFF, strlen(HOS_BUFF), 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -137,17 +135,15 @@ TEST(hos_write, normal)
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
//CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info[0]);
size_t fd1 = hos_open_fd(HOS_BUCKET, "object_append", hos_write_append_cb, (void *)"object_append", 1);
EXPECT_EQ(fd1, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[1]);
expect_fd_info[1].callback = (void *)hos_write_append_cb;
expect_fd_info[1].object = (char *)"object_append";
expect_fd_info[1].userdata = (void *)"object_append";
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
ret = hos_write(fd1, HOS_BUFF, strlen(HOS_BUFF), 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -158,8 +154,8 @@ TEST(hos_write, normal)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[1].cache_rest -= strlen(HOS_BUFF);
expect_fd_info[1].cache_count--;
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
EXPECT_TRUE(((hos_fd_context_t *)fd1)->cache != NULL);
ret = hos_write(fd1, HOS_BUFF, strlen(HOS_BUFF), 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -170,8 +166,8 @@ TEST(hos_write, normal)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[1].cache_rest -= strlen(HOS_BUFF);
expect_fd_info[1].cache_count--;
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
EXPECT_TRUE(((hos_fd_context_t *)fd1)->cache != NULL);
ret = hos_close_fd(fd, 0);
@@ -205,7 +201,6 @@ TEST(hos_write, normal)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_write, bucket_not_exits)
@@ -224,7 +219,6 @@ TEST(hos_write, bucket_not_exits)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
size_t fd = hos_open_fd("bucket_not_exits", "object_buff", hos_bucket_not_exits_cb, (void *)"object_buff", 0);
EXPECT_EQ(fd, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[0]);
@@ -232,7 +226,7 @@ TEST(hos_write, bucket_not_exits)
expect_fd_info[0].userdata = (void *)"object_buff";
expect_fd_info[0].bucket = (char *)"bucket_not_exits";
expect_fd_info[0].object = (char *)"object_buff";
CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info[0]);
CheckStructGHosFdContext((hos_fd_context_t *)fd, &expect_fd_info[0]);
int ret = hos_write(fd, HOS_BUFF, strlen(HOS_BUFF), 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -243,7 +237,6 @@ TEST(hos_write, bucket_not_exits)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
size_t fd1 = hos_open_fd("bucket_not_exits", "object_append", hos_bucket_not_exits_cb, (void *)"object_append", 1);
EXPECT_EQ(fd1, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[1]);
@@ -251,7 +244,7 @@ TEST(hos_write, bucket_not_exits)
expect_fd_info[1].userdata = (void *)"object_append";
expect_fd_info[1].bucket = (char *)"bucket_not_exits";
expect_fd_info[1].object = (char *)"object_append";
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
ret = hos_write(fd1, HOS_BUFF, strlen(HOS_BUFF), 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -262,8 +255,8 @@ TEST(hos_write, bucket_not_exits)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[1].cache_rest -= strlen(HOS_BUFF);
expect_fd_info[1].cache_count--;
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
EXPECT_TRUE(((hos_fd_context_t *)fd1)->cache != NULL);
ret = hos_write(fd1, HOS_BUFF, strlen(HOS_BUFF), 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -274,8 +267,8 @@ TEST(hos_write, bucket_not_exits)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[1].cache_rest -= strlen(HOS_BUFF);
expect_fd_info[1].cache_count--;
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
EXPECT_TRUE(((hos_fd_context_t *)fd1)->cache != NULL);
ret = hos_close_fd(fd, 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -308,7 +301,6 @@ TEST(hos_write, bucket_not_exits)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_write, sync_mode)
@@ -328,12 +320,11 @@ TEST(hos_write, sync_mode)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
size_t fd = hos_open_fd(HOS_BUCKET, "object_buff", NULL, NULL, 0);
EXPECT_EQ(fd, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[0]);
expect_fd_info[0].object = (char *)"object_buff";
CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info[0]);
CheckStructGHosFdContext((hos_fd_context_t *)fd, &expect_fd_info[0]);
int ret = hos_write(fd, HOS_BUFF, strlen(HOS_BUFF), 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -342,15 +333,13 @@ TEST(hos_write, sync_mode)
data_info->cache[0] += strlen(HOS_BUFF);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
//CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info[0]);
size_t fd1 = hos_open_fd(HOS_BUCKET, "object_append", NULL, NULL, 1);
EXPECT_EQ(fd1, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[1]);
expect_fd_info[1].object = (char *)"object_append";
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
ret = hos_write(fd1, HOS_BUFF, strlen(HOS_BUFF), 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -361,8 +350,8 @@ TEST(hos_write, sync_mode)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[1].cache_count--;
expect_fd_info[1].cache_rest -= strlen(HOS_BUFF);
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
EXPECT_TRUE(((hos_fd_context_t *)fd1)->cache != NULL);
ret = hos_write(fd1, HOS_BUFF, strlen(HOS_BUFF), 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -373,8 +362,8 @@ TEST(hos_write, sync_mode)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[1].cache_count--;
expect_fd_info[1].cache_rest -= strlen(HOS_BUFF);
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
EXPECT_TRUE(((hos_fd_context_t *)fd1)->cache != NULL);
ret = hos_close_fd(fd, 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -407,7 +396,6 @@ TEST(hos_write, sync_mode)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_write, sync_mode_bucket_not_exits)
@@ -427,13 +415,12 @@ TEST(hos_write, sync_mode_bucket_not_exits)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
size_t fd = hos_open_fd(HOS_CONF, "object_buff", NULL, NULL, 0);
EXPECT_EQ(fd, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[0]);
expect_fd_info[0].object = (char *)"object_buff";
expect_fd_info[0].bucket = (char *)HOS_CONF;
CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info[0]);
CheckStructGHosFdContext((hos_fd_context_t *)fd, &expect_fd_info[0]);
int ret = hos_write(fd, HOS_BUFF, strlen(HOS_BUFF), 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -443,16 +430,14 @@ TEST(hos_write, sync_mode_bucket_not_exits)
data_info->cache[0] += strlen(HOS_BUFF);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
//CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info[0]);
size_t fd1 = hos_open_fd(HOS_CONF, "object_append", NULL, NULL, 1);
EXPECT_EQ(fd1, 1);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info[1]);
expect_fd_info[1].object = (char *)"object_append";
expect_fd_info[1].bucket = (char *)HOS_CONF;
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
ret = hos_write(fd1, HOS_BUFF, strlen(HOS_BUFF), 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -463,8 +448,8 @@ TEST(hos_write, sync_mode_bucket_not_exits)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[1].cache_count--;
expect_fd_info[1].cache_rest -= strlen(HOS_BUFF);
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
EXPECT_TRUE(((hos_fd_context_t *)fd1)->cache != NULL);
ret = hos_write(fd1, HOS_BUFF, strlen(HOS_BUFF), 1);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -475,8 +460,8 @@ TEST(hos_write, sync_mode_bucket_not_exits)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[1].cache_count--;
expect_fd_info[1].cache_rest -= strlen(HOS_BUFF);
CheckStructGHosFdContext(g_fd_context[1], &expect_fd_info[1]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd1, &expect_fd_info[1]);
EXPECT_TRUE(((hos_fd_context_t *)fd1)->cache != NULL);
ret = hos_close_fd(fd, 0);
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -509,7 +494,6 @@ TEST(hos_write, sync_mode_bucket_not_exits)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_write, paramer_error)
@@ -525,20 +509,18 @@ TEST(hos_write, paramer_error)
gtest_hos_handle_init(&expect_hos_handle, thread_num);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
int fd = hos_open_fd(HOS_BUCKET, "object_buff", hos_callback, NULL, 0);
EXPECT_EQ(fd, 1);
long fd = hos_open_fd(HOS_BUCKET, "object_buff", hos_callback, NULL, 0);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info);
expect_fd_info.object = (char *)"object_buff";
expect_fd_info.callback = (void *)hos_callback;
CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info);
CheckStructGHosFdContext((hos_fd_context_t *)fd, &expect_fd_info);
int ret = hos_write(fd, NULL, strlen(HOS_BUFF), 0);
EXPECT_EQ(ret, HOS_PARAMETER_ERROR);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
//CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info);
ret = hos_shutdown_instance();
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -557,9 +539,9 @@ TEST(hos_write, paramer_error)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
#if 0
TEST(hos_write, fd_not_find)
{
hos_instance_s expect_hos_instance;
@@ -577,7 +559,6 @@ TEST(hos_write, fd_not_find)
EXPECT_EQ(ret, HOS_HASH_NOT_FIND);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
CheckStructGHosFdContext(g_fd_context[0], NULL);
ret = hos_shutdown_instance();
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -596,8 +577,8 @@ TEST(hos_write, fd_not_find)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
#endif
TEST(hos_write, over_threadnums)
{
@@ -612,19 +593,17 @@ TEST(hos_write, over_threadnums)
gtest_hos_handle_init(&expect_hos_handle, thread_num);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
int fd = hos_open_fd(HOS_BUCKET, "object", NULL, NULL, 0);
EXPECT_EQ(fd, 1);
long fd = hos_open_fd(HOS_BUCKET, "object", NULL, NULL, 0);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
gtest_hos_fd_init(&expect_fd_info);
expect_fd_info.object = (char *)"object";
CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info);
CheckStructGHosFdContext((hos_fd_context_t *)fd, &expect_fd_info);
int ret = hos_write(3, HOS_BUFF, strlen(HOS_CONF), 6);
EXPECT_EQ(ret, HOS_PARAMETER_ERROR);
CheckHosInstance(hos_instance, &expect_hos_instance);
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
//CheckStructGHosFdContext(g_fd_context[0], &expect_fd_info);
ret = hos_shutdown_instance();
EXPECT_EQ(ret, HOS_CLIENT_OK);
@@ -643,7 +622,6 @@ TEST(hos_write, over_threadnums)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
}
TEST(hos_write, not_init_instance)
@@ -652,6 +630,22 @@ TEST(hos_write, not_init_instance)
EXPECT_EQ(ret, HOS_INSTANCE_NOT_INIT);
}
static void read_file(char *path, char **content, int *len)
{
FILE *fp;
fp = fopen(path, "rb");
fseek(fp, 0L, SEEK_END);
size_t flen = ftell(fp);
*content = (char *)malloc(flen + 1);
fseek(fp, 0L, SEEK_SET);
fread(*content, flen, 1, fp);
(*content)[flen] = 0;
*len = flen;
fclose(fp);
}
static void *hos_function(void *ptr)
{
#define HOS_FD_NUMS_LOCAL 20
@@ -659,7 +653,7 @@ static void *hos_function(void *ptr)
size_t thread_id = reinterpret_cast<size_t>(ptr);
hos_instance hos_instance = NULL;
int i = 0;
int fd[HOS_FD_NUMS_LOCAL] = {0};
long fd[HOS_FD_NUMS_LOCAL] = {0};
char object[HOS_FD_NUMS_LOCAL][1024];
int ret = 0;
hos_instance_s expect_hos_instance;
@@ -692,12 +686,18 @@ static void *hos_function(void *ptr)
gtest_hos_fd_init(&expect_fd_info[thread_id][i]);
expect_fd_info[thread_id][i].object = object[i];
expect_fd_info[thread_id][i].callback = (void *)hos_callback;
CheckStructGHosFdContext(g_fd_context[thread_id], &expect_fd_info[thread_id][i]);
CheckStructGHosFdContext((hos_fd_context_t *)fd[i], &expect_fd_info[thread_id][i]);
}
for (i = 0; i < HOS_FD_NUMS_LOCAL; i++)
{
ret = hos_write(fd[i], HOS_BUFF, strlen(HOS_BUFF), i);
char *content = NULL;
char path[256];
int len = 0;
snprintf(path, 256, "../file/test%d.file", i%10);
read_file(path, &content, &len);
ret = hos_write(fd[i], content, len, i);
free(content);
EXPECT_EQ(ret, HOS_CLIENT_OK);
data_info->cache[i] = strlen(HOS_BUFF);
data_info->rx_bytes[i] = strlen(HOS_BUFF);
@@ -706,8 +706,8 @@ static void *hos_function(void *ptr)
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
expect_fd_info[thread_id][i].cache_rest -= strlen(HOS_BUFF);
expect_fd_info[thread_id][i].cache_count--;
CheckStructGHosFdContext(&g_fd_context[thread_id][i], &expect_fd_info[thread_id][i]);
EXPECT_TRUE(g_fd_context[1][0].cache != NULL);
CheckStructGHosFdContext((hos_fd_context_t *)fd[i], &expect_fd_info[thread_id][i]);
EXPECT_TRUE(((hos_fd_context_t *)fd[i])->cache != NULL);
}
for (i = 0; i < HOS_FD_NUMS_LOCAL; i++)
@@ -737,7 +737,6 @@ static void *hos_function(void *ptr)
free(data_info->tx_failed_pkts);
memset(&expect_hos_handle, 0, sizeof(hos_client_handle_s));
CheckStructGHosHandle(&g_hos_handle, &expect_hos_handle);
EXPECT_EQ((void *)g_fd_context, (void *)NULL);
pthread_exit(NULL);
#undef HOS_FD_NUMS_LOCAL

107
src/hos_client.cpp
View File

@@ -48,8 +48,6 @@ hos_client_handle_t g_hos_handle;//一个进程只允许有一个g_hos_handle
static std::mutex m_client_lock;
static std::mutex m_instance_lock;
static std::mutex m_delete_lock;
hos_fd_context_t **g_fd_context;
size_t *g_fd_info; //fd 实际从1开始,每个线程有独立的fd
static Aws::SDKOptions g_options;
static inline size_t get_current_ms()
@@ -62,7 +60,7 @@ static inline size_t get_current_ms()
static int hos_delete_fd(size_t fd, size_t thread_id)
{
std::lock_guard<std::mutex> locker(m_delete_lock);
hos_fd_context_t* context = find_context_by_fd(g_fd_context[thread_id], fd);
hos_fd_context_t* context = (hos_fd_context_t *)fd;
if (context == NULL)
{
return HOS_PARAMETER_ERROR;
@@ -83,7 +81,6 @@ static int hos_delete_fd(size_t fd, size_t thread_id)
free(context->object);
context->object = NULL;
}
HASH_DEL(g_fd_context[thread_id], context);
free(context);
return HOS_CLIENT_OK;
@@ -102,7 +99,7 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,
size_t thread_id, fd, stream_len;
sscanf(uuid.c_str(), "%lu %lu %lu", &thread_id, &fd, &stream_len);
a_fd_context = find_context_by_fd(g_fd_context[thread_id], fd);
a_fd_context = (hos_fd_context_t *)fd;
if (a_fd_context == NULL)
{
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_FATAL, __FUNCTION__,
@@ -156,13 +153,13 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,
{
//APPEND MODE 保留fd
atomic_add(&(a_fd_context->recive_cnt), 1);
if (a_fd_context->fd_status == HOS_FD_INJECT)
if (a_fd_context->fd_status == HOS_FD_CANCEL)
{
if (a_fd_context->position == a_fd_context->recive_cnt)
{
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG, __FUNCTION__,
"debug: [%s:%s] upload completed. [thread:%lu fd:%lu] delete",
a_fd_context->bucket, a_fd_context->object, thread_id, a_fd_context->fd);
a_fd_context->bucket, a_fd_context->object, thread_id, fd);
hos_delete_fd(fd, thread_id);
}
}
@@ -172,7 +169,7 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,
//完整上传 删除fd
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG, __FUNCTION__,
"debug: [%s:%s] upload completed. [thread:%lu fd:%lu] delete",
a_fd_context->bucket, a_fd_context->object, thread_id, a_fd_context->fd);
a_fd_context->bucket, a_fd_context->object, thread_id, fd);
hos_delete_fd(fd, thread_id);
}
}
@@ -239,9 +236,6 @@ static void hos_client_create()
g_hos_handle.task_num = (size_t *)calloc(hos_conf->thread_num, sizeof(size_t));
g_hos_handle.task_context = (size_t *)calloc(hos_conf->thread_num, sizeof(size_t));
g_fd_context = (hos_fd_context_t **)calloc(hos_conf->thread_num, sizeof(hos_fd_context_t *));
g_fd_info = (size_t *)calloc(hos_conf->thread_num, sizeof(size_t));
#if 0
if (g_hos_handle.hos_func.fd_thread == 0)
{
@@ -279,6 +273,10 @@ bool hos_verify_bucket(const char *bucket)
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG, "hos_verify_bucket","debug: bucket:%s exits", bucket);
return true;
}
else
{
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG, "hos_verify_bucket","debug: Get bucket list:%s", new_bucket.GetName().c_str());
}
}
}
else
@@ -477,19 +475,20 @@ static void hos_expand_fs2()
return ;
}
static int hos_putobject_async(Aws::S3::Model::PutObjectRequest& request, size_t stream_len,
size_t thread_id, size_t fd, const char *bucket, const char *object)
static int hos_putobject_async(Aws::S3::Model::PutObjectRequest& request, size_t stream_len, size_t thread_id, hos_fd_context_t **fd)
{
char buf[128];
int ret = 0;
hos_func_thread_t *hos_func = &g_hos_handle.hos_func;
data_info_t *data_info = NULL;
hos_config_t *hos_conf = &g_hos_handle.hos_config;
char *bucket = (*fd)->bucket;
char *object = (*fd)->object;
//设置回调函数
std::shared_ptr<Aws::Client::AsyncCallerContext> context =
Aws::MakeShared<Aws::Client::AsyncCallerContext>("");
sprintf(buf, "%lu %lu %lu", thread_id, fd, stream_len);
sprintf(buf, "%lu %lu %lu", thread_id, (long)*fd, stream_len);
context->SetUUID(buf);
if (hos_conf->max_request_num && hos_conf->max_request_context &&
@@ -544,11 +543,12 @@ static int hos_putobject_async(Aws::S3::Model::PutObjectRequest& request, size_t
}
}
static int hos_putobject_sync(Aws::S3::Model::PutObjectRequest& request, size_t stream_len, size_t thread_id, size_t fd,
const char *bucket, const char *object)
static int hos_putobject_sync(Aws::S3::Model::PutObjectRequest& request, size_t stream_len, size_t thread_id, hos_fd_context_t **fd)
{
hos_func_thread_t *hos_func = &g_hos_handle.hos_func;
data_info_t *data_info = NULL;
char *bucket = (*fd)->bucket;
char *object = (*fd)->object;
auto& S3Client = *(g_hos_handle.S3Client);
Aws::S3::Model::PutObjectOutcome Outcome = S3Client.PutObject(request);
@@ -748,7 +748,7 @@ static int hos_upload_stream(const char *bucket, const char *object, const char
request.SetKey(object);
//设置上传数据类型
if (mode == 0)
if (mode == FILE_MODE)
{
//文件类型
const std::shared_ptr<Aws::IOStream> input_data =
@@ -776,18 +776,22 @@ static int hos_upload_stream(const char *bucket, const char *object, const char
}
//设置回调函数
size_t fd = ++g_fd_info[thread_id];
hos_fd_context_t info = {fd, 0, (char *)bucket, (char *)object, (void *)callback, userdata, NULL, 0, 0, 0 };
add_fd_context(&g_fd_context[thread_id], &info);
hos_fd_context_t *hos_fd = (hos_fd_context_t *)calloc(1, sizeof(hos_fd_context_t));
hos_fd->mode = mode;
hos_fd->bucket = (char *)malloc(strlen(bucket) + 1);
memcpy(hos_fd->bucket, bucket, strlen(bucket) + 1);
hos_fd->object = (char *)malloc(strlen(object) + 1);
memcpy(hos_fd->object, object, strlen(object) + 1);
hos_fd->callback = (void *)callback;
hos_fd->userdata = userdata;
if (hos_conf->pool_thread_size > 0)
{
ret = hos_putobject_async(request, data_len, thread_id, fd, bucket, object);
ret = hos_putobject_async(request, data_len, thread_id, &hos_fd);
}
else
{
ret = hos_putobject_sync(request, data_len, thread_id, fd, bucket, object);
ret = hos_putobject_sync(request, data_len, thread_id, &hos_fd);
}
return ret;
@@ -835,7 +839,7 @@ int hos_upload_buf(const char *bucket, const char *object, const char *buf, size
return hos_upload_stream(bucket, object, buf, buf_len, callback, userdata, thread_id);
}
int hos_open_fd(const char *bucket, const char *object, put_finished_callback callback, void *userdata, size_t thread_id)
long hos_open_fd(const char *bucket, const char *object, put_finished_callback callback, void *userdata, size_t thread_id)
{
if (g_hos_instance.result == false || g_hos_handle.S3Client == NULL)
{
@@ -849,15 +853,22 @@ int hos_open_fd(const char *bucket, const char *object, put_finished_callback ca
return HOS_PARAMETER_ERROR;
}
size_t fd = ++g_fd_info[thread_id];
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG, "hos_open_fd", "debug: thread_id:%lu, fd:%lu", thread_id, fd);
hos_fd_context_t *hos_fd = (hos_fd_context_t *)calloc(1, sizeof(hos_fd_context_t));
hos_fd->mode = BUFF_MODE | APPEND_MODE;
hos_fd->bucket = (char *)malloc(strlen(bucket) + 1);
memcpy(hos_fd->bucket, bucket, strlen(bucket) + 1);
hos_fd->object = (char *)malloc(strlen(object) + 1);
memcpy(hos_fd->object, object, strlen(object) + 1);
hos_fd->callback = (void *)callback;
hos_fd->userdata = userdata;
hos_fd->cache_count = g_hos_handle.hos_config.cache_count;
hos_fd->cache_rest = g_hos_handle.hos_config.cache_size;
hos_fd->fd_status = HOS_FD_REGISTER;
hos_fd->reslut = true;
hos_fd_context_t info = {fd, BUFF_MODE | APPEND_MODE, (char *)bucket, (char *)object, (void *)callback, userdata,
NULL,/*cache*/ g_hos_handle.hos_config.cache_count, 0,/*position*/ 0,/*recive_cnt*/
(long)g_hos_handle.hos_config.cache_size,/*cache_rest*/ HOS_FD_REGISTER,/*fd_status*/};
add_fd_context(&g_fd_context[thread_id], &info);
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG, "hos_open_fd", "debug: thread_id:%lu, fd:%lu", thread_id, (long)&hos_fd);
return fd;
return (long)hos_fd;
}
int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id)
@@ -883,7 +894,7 @@ int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id
return HOS_PARAMETER_ERROR;
}
a_fd_context = find_context_by_fd(g_fd_context[thread_id], fd);
a_fd_context = (hos_fd_context_t *)fd;
if (a_fd_context == NULL)
{
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_FATAL, __FUNCTION__, "error: fd info not find. thread_id:%lu, fd:%lu", thread_id, fd);
@@ -943,11 +954,11 @@ int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id
if (hos_conf->pool_thread_size > 0)
{
ret = hos_putobject_async(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object);
ret = hos_putobject_async(request, upload_len, thread_id, &a_fd_context);
}
else
{
ret = hos_putobject_sync(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object);
ret = hos_putobject_sync(request, upload_len, thread_id, &a_fd_context);
}
//恢复fd 的cache设置
@@ -983,7 +994,7 @@ int hos_close_fd(size_t fd, size_t thread_id)
fd, thread_id, hos_conf->thread_num);
return HOS_PARAMETER_ERROR;
}
a_fd_context = find_context_by_fd(g_fd_context[thread_id], fd);
a_fd_context = (hos_fd_context_t *)fd;
if (a_fd_context == NULL)
{
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG,
@@ -1016,18 +1027,18 @@ int hos_close_fd(size_t fd, size_t thread_id)
if (hos_conf->pool_thread_size > 0)
{
hos_putobject_async(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object);
hos_putobject_async(request, upload_len, thread_id, &a_fd_context);
}
else
{
hos_putobject_sync(request, upload_len, thread_id, fd, a_fd_context->bucket, a_fd_context->object);
hos_putobject_sync(request, upload_len, thread_id, &a_fd_context);
}
data_info_t *data_info = (data_info_t *)(g_hos_handle.hos_func.fs2_info.reserved);
if (data_info)
data_info->cache[thread_id] = 0;
}
}
a_fd_context->fd_status = HOS_FD_INJECT;
a_fd_context->fd_status = HOS_FD_CANCEL;
a_fd_context->cache.reset();
a_fd_context->cache = NULL;
a_fd_context->cache_rest = hos_conf->cache_size;
@@ -1046,7 +1057,7 @@ int hos_close_fd(size_t fd, size_t thread_id)
{
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG, __FUNCTION__,
"debug: [%s:%s] upload completed. [thread:%lu fd:%lu] delete",
a_fd_context->bucket, a_fd_context->object, thread_id, a_fd_context->fd);
a_fd_context->bucket, a_fd_context->object, thread_id, fd);
hos_delete_fd(fd, thread_id);
}
}
@@ -1058,7 +1069,6 @@ int hos_shutdown_instance()
{
std::lock_guard<std::mutex> locker(m_instance_lock);
size_t i = 0;
hos_config_t *hos_conf = &g_hos_handle.hos_config;
hos_func_thread_t *hos_func = &g_hos_handle.hos_func;
size_t task_num = 0;
@@ -1153,23 +1163,6 @@ int hos_shutdown_instance()
}
MESA_HANDLE_RUNTIME_LOG(g_hos_handle.log, RLOG_LV_DEBUG, __FUNCTION__, "debug: delete s3client.");
if (g_fd_info)
{
free(g_fd_info);
g_fd_info = NULL;
}
for (i = 0; i < hos_conf->thread_num; i++)
{
delete_all(&g_fd_context[i]);
}
if (g_fd_context)
{
free(g_fd_context);
g_fd_context = NULL;
}
Aws::ShutdownAPI(g_options);
MESA_destroy_runtime_log_handle(g_hos_handle.log);
g_hos_handle.log = NULL;

4
src/hos_client.h
View File

@@ -129,9 +129,9 @@ int hos_upload_buf(const char *bucket, const char *object, const char *buf, size
* void *data 用户回调函数自定义数据
* size_t thread_id 线程ID
* int mode 模式 (FILE OR BUFFER, APPEND OR NOT)
* 返回值 int 成功返回fd(fd >=3)，失败返回hoserros错误码
* 返回值 long 成功返回fd(fd >0)，失败返回hoserros错误码
*************************************************************************************/
int hos_open_fd(const char *bucket, const char *object, put_finished_callback callback, void *userdata, size_t thread_id);
long hos_open_fd(const char *bucket, const char *object, put_finished_callback callback, void *userdata, size_t thread_id);
/*************************************************************************************
* 函数名： hos_write
* 参数： size_t fd hos_open_fd返回的fd

24
src/hos_common.h
View File

@@ -4,7 +4,6 @@
#include <netinet/in.h>
#include <mutex>
#include "field_stat2.h"
#include "hos_hash.h"
#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/core/auth/AWSCredentials.h>
@@ -103,9 +102,28 @@ typedef struct hos_client_handle_s
size_t *task_context;
}hos_client_handle_t;
typedef struct hos_fd_context_s
{
int mode;
//hos_client_handle handle;
char *bucket;
char *object;
void *callback;
void *userdata;
std::shared_ptr<Aws::IOStream> cache;
size_t cache_count;
size_t position;
size_t recive_cnt;
long cache_rest;
int fd_status;
#define HOS_FD_REGISTER 0
#define HOS_FD_CANCEL 1
bool reslut; /*PutObjectAsync result*/
const char *error; /*PutObjectAsync error message*/
}hos_fd_context_t;
extern struct hos_instance_s g_hos_instance;
extern hos_client_handle_t g_hos_handle;//一个进程只允许有一个g_hos_handle
extern hos_fd_context_t **g_fd_context;
extern size_t *g_fd_info; //fd 实际从1开始,每个线程有独立的fd
#endif

97
src/hos_hash.cpp
View File

@@ -1,97 +0,0 @@
/*************************************************************************
> File Name: uthash.cpp
> Author: pxz
> Created Time: Fri 18 Sep 2020 04:26:09 PM CST
************************************************************************/
#include "hos_hash.h"
void add_fd_context(hos_fd_context_t **handle, hos_fd_context_t *input)
{
hos_fd_context_t *value = NULL;
HASH_FIND(hh,*handle,&input->fd,sizeof(input->fd),value);
if (value == NULL)
{
value = (hos_fd_context_t *)malloc(sizeof(hos_fd_context_t));
memcpy(value, input, sizeof(hos_fd_context_t));
value->object = (char *)calloc(1, strlen(input->object) + 1);
value->bucket = (char *)calloc(1, strlen(input->bucket) + 1);
memcpy(value->bucket, input->bucket, strlen(input->bucket));
memcpy(value->object, input->object, strlen(input->object));
HASH_ADD(hh,*handle,fd,sizeof(long),value);
}
else
{
value->mode = input->mode;
if (value->object != NULL)
{
free(value->object);
value->object = NULL;
}
if (value->bucket != NULL)
{
free(value->bucket);
value->bucket = NULL;
}
value->object = (char *)calloc(1, strlen(input->object) + 1);
value->bucket = (char *)calloc(1, strlen(input->bucket) + 1);
memcpy(value->bucket, input->bucket, strlen(input->bucket));
memcpy(value->object, input->object, strlen(input->object));
value->callback = input->callback;
value->userdata = input->userdata;
value->cache = input->cache;
value->cache_count = input->cache_count;
value->cache_rest = input->cache_rest;
value->position = input->position;
value->recive_cnt = input->recive_cnt;
value->fd_status = value->fd_status;
}
}
hos_fd_context_t *find_context_by_fd(hos_fd_context_t *handle, size_t fd)
{
hos_fd_context_t *value = NULL;
HASH_FIND(hh,handle,&fd,sizeof(long),value);
return value;
}
void delete_context_by_fd(hos_fd_context_t **handle, size_t fd)
{
hos_fd_context_t *value = NULL;
HASH_FIND(hh,*handle,&fd,sizeof(long),value);
if (value)
{
if (value->bucket)
{
free(value->bucket);
value->bucket = NULL;
}
if (value->object)
{
free(value->object);
value->object = NULL;
}
HASH_DEL(*handle, value);
free(value);
}
}
void delete_all(hos_fd_context_t **handle)
{
hos_fd_context_t *current, *tmp;
HASH_ITER(hh, *handle, current, tmp)
{
if (current->bucket)
{
free(current->bucket);
current->bucket = NULL;
}
if (current->object)
{
free(current->object);
current->object = NULL;
}
HASH_DEL(*handle, current);
free(current);
}
}

40
src/hos_hash.h
View File

@@ -1,40 +0,0 @@
/*************************************************************************
> File Name: hos_hash.h
> Author: pxz
> Created Time: Fri 18 Sep 2020 05:00:04 PM CST
************************************************************************/
#ifndef __HOS_HASH_H__
#define __HOS_HASH_H__
#include <aws/core/Aws.h>
//#include "hos_client.h"
#include "uthash.h"
typedef struct hos_fd_context_s
{
size_t fd;
int mode;
//hos_client_handle handle;
char *bucket;
char *object;
void *callback;
void *userdata;
std::shared_ptr<Aws::IOStream> cache;
size_t cache_count;
size_t position;
size_t recive_cnt;
long cache_rest;
int fd_status;
#define HOS_FD_REGISTER 0
#define HOS_FD_INJECT 1
bool reslut; /*PutObjectAsync result*/
const char *error; /*PutObjectAsync error message*/
UT_hash_handle hh;
}hos_fd_context_t;
void add_fd_context(hos_fd_context_t **handle, hos_fd_context_t *input);
hos_fd_context_t *find_context_by_fd(hos_fd_context_t *handle, size_t fd);
void delete_context_by_fd(hos_fd_context_t **handle, size_t fd);
void delete_all(hos_fd_context_t **handle);
#endif

1150
src/uthash.h
View File

File diff suppressed because it is too large Load Diff