gfwleak/pxz-hos-client-cpp-module
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复add_hos_info导致的segment fault

Browse Source
This commit is contained in:
彭宣正
2021-06-02 15:49:24 +08:00
parent 20e6f942cd
commit da3bc1db06
7 changed files with 69 additions and 22137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22113
.gitignore vendored
View File

File diff suppressed because it is too large Load Diff

2
example/demo/hos_upload_complete.cpp
View File

@@ -82,7 +82,7 @@ int main(int argc, char *argv[])
file_to_buffer(object, buf, &buf_size); file_to_buffer(object, buf, &buf_size);
debuginfo("hos_client_init start ...\n"); debuginfo("hos_client_init start ...\n");
hos_client_handle handle = hos_client_create("192.168.44.10", 9098, "default", "default", 3000); hos_client_handle handle = hos_client_create("192.168.44.12", 9098, "default", "default", 3000);
if (handle == NULL) if (handle == NULL)
{ {
debuginfo("error:hos_client_handle\n"); debuginfo("error:hos_client_handle\n");

16
example/demo/hos_write_complete.cpp
View File

@@ -15,7 +15,7 @@ extern "C"
//#define test_times 10000 //#define test_times 10000
#define debuginfo (void) #define debuginfo printf
typedef struct userdata_s typedef struct userdata_s
{ {
@@ -59,7 +59,7 @@ void callback(bool result, const char *error, const char *bucket, const char *ob
int main(int argc, char *argv[]) int main(int argc, char *argv[])
{ {
if (argc != 4) if (argc != 3)
{ {
debuginfo("usege: singThread [bucket name] [object name]\n"); debuginfo("usege: singThread [bucket name] [object name]\n");
return -1; return -1;
@@ -69,7 +69,6 @@ int main(int argc, char *argv[])
int i = 0; int i = 0;
char *bucket = argv[1]; char *bucket = argv[1];
char *object = argv[2]; char *object = argv[2];
int test_times = atoi(argv[3]);
char *buf = (char *)malloc(1024 * 1024 * 40); char *buf = (char *)malloc(1024 * 1024 * 40);
size_t buf_size; size_t buf_size;
int mode = FILE_MODE; int mode = FILE_MODE;
@@ -79,7 +78,7 @@ int main(int argc, char *argv[])
file_to_buffer(object, buf, &buf_size); file_to_buffer(object, buf, &buf_size);
debuginfo("hos_client_init start ...\n"); debuginfo("hos_client_init start ...\n");
hos_client_handle handle = hos_client_create("192.168.40.223", 9098, "default", "default", 400); hos_client_handle handle = hos_client_create("192.168.44.12", 9098, "default", "default", 100);
//hos_client_handle handle = hos_client_create("http://192.168.32.10:9098/hos/", "default", "default", 4); //hos_client_handle handle = hos_client_create("http://192.168.32.10:9098/hos/", "default", "default", 4);
if (handle == NULL) if (handle == NULL)
{ {
@@ -108,14 +107,10 @@ int main(int argc, char *argv[])
mode = FILE_MODE; mode = FILE_MODE;
debuginfo("hos_upload_file start ...\n"); debuginfo("hos_upload_file start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < test_times; i++) fd[0] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
{ hos_write(fd[0], object, 0, 0);
fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
hos_write(fd[i], object, 0, 0);
}
clock_gettime(CLOCK_MONOTONIC, &end); clock_gettime(CLOCK_MONOTONIC, &end);
time = calc_time(start, end); time = calc_time(start, end);
time /= test_times;
printf("hos_upload_file spent %llu ns\n", time); printf("hos_upload_file spent %llu ns\n", time);
debuginfo("hos_upload_file end ...\n"); debuginfo("hos_upload_file end ...\n");
#else #else
@@ -151,7 +146,6 @@ int main(int argc, char *argv[])
//printf("start:%lu\n", start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec); //printf("start:%lu\n", start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec);
//printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec); //printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec);
time = calc_time(start, finished); time = calc_time(start, finished);
time /= test_times;
printf("hos upload finished spent %llu ns\n", time); printf("hos upload finished spent %llu ns\n", time);
} }

35
example/performance/HosClientPerformance.cpp
View File

@@ -69,8 +69,7 @@ typedef struct userdata_s
static size_t calc_time(struct timespec start, struct timespec end) static size_t calc_time(struct timespec start, struct timespec end)
{ {
return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - return (end.tv_sec - start.tv_sec) * 1000 * 1000 * 1000 + end.tv_nsec - start.tv_nsec;
(start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
} }
int read_file_list(const char *path, char file_name[][256]) int read_file_list(const char *path, char file_name[][256])
@@ -216,7 +215,7 @@ static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t
{ {
int i = 0; int i = 0;
int j = 0; int j = 0;
size_t fd = 0; size_t fd[1000] = {0};
size_t tmp = 0; size_t tmp = 0;
size_t rest = 0; size_t rest = 0;
struct timespec tstart, ttmp; struct timespec tstart, ttmp;
@@ -230,11 +229,11 @@ static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t
int drop = 0; int drop = 0;
double variance = 0.00; double variance = 0.00;
double average = 0.00; double average = 0.00;
long record[1000] = {0}; long record[30000] = {0};
if (conf->slice) if (conf->slice)
{ {
fd = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); fd[0] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
for (i = 0; i < test_times; i++) for (i = 0; i < test_times; i++)
{ {
j = 0; j = 0;
@@ -245,10 +244,10 @@ static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t
rest = buff_len - tmp; rest = buff_len - tmp;
if (rest < conf->slice) if (rest < conf->slice)
{ {
hos_write(fd, &buff[tmp], rest, thread_info->thread_num); hos_write(fd[0], &buff[tmp], rest, thread_info->thread_num);
break; break;
} }
hos_write(fd, &buff[tmp], conf->slice, thread_info->thread_num); hos_write(fd[0], &buff[tmp], conf->slice, thread_info->thread_num);
j++; j++;
} }
clock_gettime(CLOCK_MONOTONIC, &ttmp); clock_gettime(CLOCK_MONOTONIC, &ttmp);
@@ -272,7 +271,7 @@ static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t
if (conf->mode & APPEND_MODE) if (conf->mode & APPEND_MODE)
{ {
fd = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); fd[0] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
#if 1 #if 1
for (i = 0; i < test_times; i++) for (i = 0; i < test_times; i++)
{ {
@@ -285,10 +284,10 @@ static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t
rest = buff_len - tmp; rest = buff_len - tmp;
if (rest < conf->append_size) if (rest < conf->append_size)
{ {
hos_write(fd, &buff[tmp], rest, thread_info->thread_num); hos_write(fd[0], &buff[tmp], rest, thread_info->thread_num);
break; break;
} }
hos_write(fd, &buff[tmp], conf->append_size, thread_info->thread_num); hos_write(fd[0], &buff[tmp], conf->append_size, thread_info->thread_num);
j++; j++;
} }
#if 1 #if 1
@@ -315,14 +314,14 @@ static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t
for (i = 0; i < test_times; i++) for (i = 0; i < test_times; i++)
{ {
clock_gettime(CLOCK_MONOTONIC, &tstart); clock_gettime(CLOCK_MONOTONIC, &tstart);
fd = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); fd[i] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
ret = hos_write(fd, buff, buff_len, thread_info->thread_num); ret = hos_write(fd[i], buff, buff_len, thread_info->thread_num);
if (ret == HOS_CLIENT_OK) if (ret == HOS_CLIENT_OK)
{ {
success_cnt++; success_cnt++;
}else }else
{ {
printf("error code:%d, thread_id:%d\n", ret, thread_info->thread_num); //printf("error code:%d, thread_id:%d\n", ret, thread_info->thread_num);
//break; //break;
} }
clock_gettime(CLOCK_MONOTONIC, &ttmp); clock_gettime(CLOCK_MONOTONIC, &ttmp);
@@ -348,7 +347,13 @@ static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t
} }
} }
hos_close_fd(fd, thread_info->thread_num); for (i = 0; i < test_times; i++)
{
if (fd[i] > 2)
{
hos_close_fd(fd[i], thread_info->thread_num);
}
}
return 0; return 0;
} }
@@ -365,7 +370,7 @@ static void *put_object_thread(void *ptr)
char *buff = NULL; char *buff = NULL;
char file_name[100][256]; char file_name[100][256];
struct stat s_buf; struct stat s_buf;
int test_times = 100; int test_times = 1000;
buff = (char *)malloc(30 * 1024 * 1024); buff = (char *)malloc(30 * 1024 * 1024);
if (buff == NULL) if (buff == NULL)

8
example/performance/test.sh
View File

@@ -10,7 +10,7 @@ size=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
num=0 num=0
while((${num} < 37)) while((${num} < 100))
do do
#./HosClientPerformance -a ${size[$num]} #./HosClientPerformance -a ${size[$num]}
#echo "./HosClientPerformance -a ${size[$num]} -A -f ./file/100k.data" #echo "./HosClientPerformance -a ${size[$num]} -A -f ./file/100k.data"
@@ -19,7 +19,9 @@ do
#./HosClientPerformance -f ./file/${size[$num]}.data -F #./HosClientPerformance -f ./file/${size[$num]}.data -F
#echo ./HosClientPerformance -a 100 -A -f ./file/100k.data -t ${size[$num]} -S 1024 #echo ./HosClientPerformance -a 100 -A -f ./file/100k.data -t ${size[$num]} -S 1024
#./HosClientPerformance -a 100 -A -f ./file/100k.data -t ${size[$num]} -S 1024 #./HosClientPerformance -a 100 -A -f ./file/100k.data -t ${size[$num]} -S 1024
echo ./HosClientPerformance -a 100 -A -f ./file/100k.data -S ${size[$num]} -t 32 #echo ./HosClientPerformance -a 100 -A -f ./file/100k.data -S ${size[$num]} -t 32
./HosClientPerformance -a 100 -A -f ./file/100k.data -S ${size[$num]} -t 32 #./HosClientPerformance -a 100 -A -f ./file/100k.data -S ${size[$num]} -t 32
echo ./HosClientPerformance -P1 -t32
./HosClientPerformance -P1 -t32
let "num++" let "num++"
done done

4
src/hos_client.cpp
View File

@@ -221,6 +221,7 @@ void hos_set_thread_sum(hos_client_handle client, size_t thread_sum)
if (fd_info) if (fd_info)
{ {
fd_info = (size_t (*) [MAX_HOS_CLIENT_FD_NUM + 1])realloc(fd_info, thread_sum * sizeof(size_t [MAX_HOS_CLIENT_FD_NUM + 1])); fd_info = (size_t (*) [MAX_HOS_CLIENT_FD_NUM + 1])realloc(fd_info, thread_sum * sizeof(size_t [MAX_HOS_CLIENT_FD_NUM + 1]));
memset(&fd_info[client->thread_sum], 0, (thread_sum - client->thread_sum) * sizeof(size_t [MAX_HOS_CLIENT_FD_NUM + 1]));
}else }else
{ {
fd_info = (size_t (*) [MAX_HOS_CLIENT_FD_NUM + 1])calloc(thread_sum, sizeof(size_t [MAX_HOS_CLIENT_FD_NUM + 1])); fd_info = (size_t (*) [MAX_HOS_CLIENT_FD_NUM + 1])calloc(thread_sum, sizeof(size_t [MAX_HOS_CLIENT_FD_NUM + 1]));
@@ -743,7 +744,7 @@ static void *hos_fd_manage(void *ptr)
int hos_open_fd(hos_client_handle handle, const char *bucket, const char *object, put_finished_callback callback, void *userdata, size_t thread_id, int mode) int hos_open_fd(hos_client_handle handle, const char *bucket, const char *object, put_finished_callback callback, void *userdata, size_t thread_id, int mode)
{ {
if ((handle == NULL) || (bucket == NULL) || (object == NULL) || (thread_id > handle->thread_sum)) if ((handle == NULL) || (bucket == NULL) || (object == NULL) || (thread_id > handle->thread_sum) || strlen(bucket) == 0 || strlen(object) == 0)
{ {
return HOS_PARAMETER_ERROR; return HOS_PARAMETER_ERROR;
} }
@@ -760,6 +761,7 @@ int hos_open_fd(hos_client_handle handle, const char *bucket, const char *object
#if 1 #if 1
if (handle->fd_thread == 0) if (handle->fd_thread == 0)
{ {
std::lock_guard<std::mutex> locker(m_client_lock);
handle->fd_thread_status = 0; handle->fd_thread_status = 0;
pthread_create(&handle->fd_thread, NULL, hos_fd_manage, handle); pthread_create(&handle->fd_thread, NULL, hos_fd_manage, handle);
} }

28
src/hos_hash.cpp
View File

@@ -13,18 +13,30 @@ void add_hos_info(hos_info_t **handle, hos_info_t *input)
{ {
value = (hos_info_t *)malloc(sizeof(hos_info_t)); value = (hos_info_t *)malloc(sizeof(hos_info_t));
memcpy(value, input, sizeof(hos_info_t)); memcpy(value, input, sizeof(hos_info_t));
value->object = (char *)malloc(strlen(input->object) + 1); value->object = (char *)calloc(1, strlen(input->object) + 1);
value->bucket = (char *)malloc(strlen(input->bucket) + 1); value->bucket = (char *)calloc(1, strlen(input->bucket) + 1);
memcpy(value->bucket, input->bucket, strlen(input->bucket) + 1); memcpy(value->bucket, input->bucket, strlen(input->bucket));
memcpy(value->object, input->object, strlen(input->object) + 1); memcpy(value->object, input->object, strlen(input->object));
HASH_ADD_INT(*handle, fd, value); HASH_ADD_INT(*handle, fd, value);
} }
else else
{ {
value->mode = input->mode; value->mode = input->mode;
value->handle = input->handle; value->handle = input->handle;
memcpy(value->bucket, input->bucket, strlen(input->bucket) + 1); if (value->object != NULL)
memcpy(value->object, input->object, strlen(input->object) + 1); {
free(value->object);
value->object = NULL;
}
if (value->bucket != NULL)
{
free(value->bucket);
value->bucket = NULL;
}
value->object = (char *)calloc(1, strlen(input->object) + 1);
value->bucket = (char *)calloc(1, strlen(input->bucket) + 1);
memcpy(value->bucket, input->bucket, strlen(input->bucket));
memcpy(value->object, input->object, strlen(input->object));
value->callback = input->callback; value->callback = input->callback;
value->userdata = input->userdata; value->userdata = input->userdata;
value->cache = input->cache; value->cache = input->cache;
@@ -55,10 +67,12 @@ void delete_info_by_fd(hos_info_t **handle, size_t fd)
if (value->bucket) if (value->bucket)
{ {
free(value->bucket); free(value->bucket);
value->bucket = NULL;
} }
if (value->object) if (value->object)
{ {
free(value->object); free(value->object);
value->object = NULL;
} }
HASH_DEL(*handle, value); HASH_DEL(*handle, value);
free(value); free(value);
@@ -73,10 +87,12 @@ void delete_all(hos_info_t **handle)
if (current->bucket) if (current->bucket)
{ {
free(current->bucket); free(current->bucket);
current->bucket = NULL;
} }
if (current->object) if (current->object)
{ {
free(current->object); free(current->object);
current->object = NULL;
} }
HASH_DEL(*handle, current); HASH_DEL(*handle, current);
free(current); free(current);