gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加HTTP Post上传配置接口，支持主从双机备份与同步（冷备）

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-25 18:40:20 +08:00
parent 67bafbefc9
commit 1aca701f12
28 changed files with 3295 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
client/doris_client_transfer.cpp
View File

@@ -50,7 +50,7 @@ void doris_http_ctx_destroy(struct doris_http_ctx *ctx)
}
struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance,
struct doris_http_callback *cb, u_int64_t balance_seed, char *host, int32_t size)
struct doris_http_callback *cb, u_int64_t balance_seed, char *host/*OUT*/, int32_t size)
{
struct doris_http_ctx *ctx;
struct doris_curl_multihd *multidata;
@@ -62,8 +62,10 @@ struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance,
}
assert(instance->server_hosts->find(result.bucket_id) != instance->server_hosts->end());
multidata = instance->server_hosts->find(result.bucket_id)->second;
snprintf(host, size, multidata->host->srvaddr);
if(host != NULL)
{
snprintf(host, size, multidata->host->srvaddr);
}
ctx = (struct doris_http_ctx *)calloc(1, sizeof(struct doris_http_ctx));
ctx->instance = instance;
ctx->multidata = multidata;
@@ -92,8 +94,8 @@ static inline void curl_set_common_options(CURL *curl, long transfer_timeout, ch
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 1000L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //<2F><><EFBFBD>Է<EFBFBD><D4B7>ֶ<EFBFBD><D6B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ij<EFBFBD><C4B3><EFBFBD>ӽ<EFBFBD><D3BD>տ<EFBFBD>ס<EFBFBD><D7A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
//ctx->error="Operation too slow. Less than 100 bytes/sec transferred the last 10 seconds"
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 10L);
//ctx->error="Operation too slow. Less than 100 bytes/sec transferred the last 30 seconds"
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "Doris Client Linux X64");
}
@@ -136,6 +138,21 @@ void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header)
ctx->headers = curl_slist_append(ctx->headers, header);
}
/*maximum length 1024*/
void doris_http_ctx_add_header_kvstr(struct doris_http_ctx *ctx, const char *headername, const char *value)
{
char header[1024];
snprintf(header, 1024, "%s: %s", headername, value);
ctx->headers = curl_slist_append(ctx->headers, header);
}
void doris_http_ctx_add_header_kvint(struct doris_http_ctx *ctx, const char *headername, u_int64_t value)
{
char header[1024];
snprintf(header, 1024, "%s: %lu", headername, value);
ctx->headers = curl_slist_append(ctx->headers, header);
}
int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri)
{
char minio_url[2048];
@@ -143,7 +160,7 @@ int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri)
assert(ctx->curl == NULL);
if(NULL == (ctx->curl=curl_easy_init()))
{
return -1;
assert(0);return -1;
}
if(ctx->instance->param->ssl_connection)
@@ -171,8 +188,7 @@ int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri)
if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
{
assert(0);
return -2;
assert(0);return -2;
}
ctx->transfering = 1;
return 0;
@@ -185,7 +201,7 @@ int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri,
assert(ctx->curl == NULL);
if(NULL == (ctx->curl=curl_easy_init()))
{
return -1;
assert(0);return -1;
}
doris_http_ctx_add_header(ctx, "Expect:"); //ע<><D7A2>POST<53><54><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Expect<63><74>ϵ<EFBFBD><CFB5>Ҫ<EFBFBD><D2AA>ȷ<EFBFBD><C8B7><EFBFBD><EFBFBD>CURLOPT_POSTFIELDSIZE
@@ -219,8 +235,153 @@ int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri,
if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
{
assert(0);
return -2;
assert(0);return -2;
}
ctx->transfering = 1;
return 0;
}
static size_t curl_put_data_request_send_cb(void *ptr, size_t size, size_t count, void *userp)
{
size_t len;
struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp;
if(size==0 || count==0 || ctx->put_offset>=ctx->put_length)
{
return 0; //<2F><>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
}
len = ctx->put_length - ctx->put_offset; //ʣ<><CAA3><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ij<EFBFBD><C4B3><EFBFBD>
if(len > size * count)
{
len = size * count;
}
memcpy(ptr, ctx->put_data + ctx->put_offset, len);
ctx->put_offset += len;
if(ctx->cb.read_process_cb != NULL)
{
ctx->cb.read_process_cb(ctx->put_data, ctx->put_offset, ctx->cb.userp);
}
return len;
}
int doris_http_launch_put_request_data(struct doris_http_ctx *ctx, const char *uri, char *data, size_t data_len)
{
char minio_url[2048];
assert(ctx->curl == NULL);
if(NULL == (ctx->curl=curl_easy_init()))
{
assert(0);return -1;
}
ctx->put_data = data;
ctx->put_length = data_len;
ctx->put_offset = 0;
if(ctx->instance->param->ssl_connection)
{
snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri);
curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L);
}
else
{
snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri);
}
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
if(ctx->cb.header_cb != NULL)
{
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
}
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put_length);
curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_data_request_send_cb);
curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
{
assert(0);return -2;
}
ctx->transfering = 1;
return 0;
}
static size_t curl_put_evbuf_request_send_cb(void *ptr, size_t size, size_t count, void *userp)
{
size_t len, space=size*count, send_len;
struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp;
if(size==0 || count==0 || ctx->put_offset>=ctx->put_length)
{
return 0;
}
len = ctx->put_length - ctx->put_offset;
if(len > space)
{
len = space;
}
send_len = evbuffer_remove(ctx->put_evbuf, ptr, len);
assert(send_len>0);
ctx->put_offset += send_len;
ctx->cb.read_process_cb(ctx->put_evbuf, ctx->put_offset, ctx->cb.userp);
return send_len;
}
int doris_http_launch_put_request_evbuf(struct doris_http_ctx *ctx, const char *uri, struct evbuffer *evbuf, size_t data_len)
{
char minio_url[2048];
assert(ctx->curl == NULL);
if(NULL == (ctx->curl=curl_easy_init()))
{
assert(0);return -1;
}
ctx->put_evbuf = evbuf;
ctx->put_length = data_len;
ctx->put_offset = 0;
if(ctx->instance->param->ssl_connection)
{
snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri);
curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L);
}
else
{
snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri);
}
curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb);
curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
if(ctx->cb.header_cb != NULL)
{
curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb);
curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
}
curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put_length);
curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_evbuf_request_send_cb);
curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
{
assert(0);return -2;
}
ctx->transfering = 1;
return 0;