gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加HTTP Post上传配置接口，支持主从双机备份与同步（冷备）

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-25 18:40:20 +08:00
parent 67bafbefc9
commit 1aca701f12
28 changed files with 3295 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
server/doris_server_http.cpp
View File

@@ -16,64 +16,11 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <event2/bufferevent_ssl.h>
#include "doris_server_main.h"
#include "doris_server_http.h"
extern struct doris_global_info g_doris_server_info;
static inline void set_sockopt_keepalive(int sd, int keepidle, int keepintvl, int keepcnt)
{
int keepalive = 1;
setsockopt(sd, SOL_SOCKET, SO_KEEPALIVE, (void*)&keepalive, sizeof(keepalive));
setsockopt(sd, SOL_TCP, TCP_KEEPIDLE, (void*)&keepidle, sizeof(keepidle));
setsockopt(sd, SOL_TCP, TCP_KEEPINTVL, (void*)&keepintvl, sizeof(keepintvl));
setsockopt(sd, SOL_TCP, TCP_KEEPCNT, (void*)&keepcnt, sizeof(keepcnt));
}
static inline void set_listen_sockopt(int sd)
{
if(g_doris_server_info.sock_recv_bufsize > 0)
{
setsockopt(sd, SOL_SOCKET, SO_RCVBUF, &g_doris_server_info.sock_recv_bufsize, sizeof(u_int32_t));
}
}
int doris_create_listen_socket(int bind_port)
{
evutil_socket_t listener;
struct sockaddr_in sin;
listener = socket(AF_INET, SOCK_STREAM, 0);
if(listener < 0)
{
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "create socket error!\n");
return -1;
}
set_sockopt_keepalive(listener, 300, 10, 2);
set_listen_sockopt(listener);
evutil_make_listen_socket_reuseable(listener);
sin.sin_family = AF_INET;
sin.sin_addr.s_addr=htonl(INADDR_ANY);
sin.sin_port = htons(bind_port);
if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0)
{
printf("bind socket to port: %d error: %s!\n", bind_port, strerror(errno));
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "bind socket to port: %d error: %s!\n", bind_port, strerror(errno));
assert(0);return -2;
}
if (listen(listener, 1024)<0)
{
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "listen socket 1024 error!\n");
return -3;
}
evutil_make_socket_nonblocking(listener);
return listener;
}
struct version_list_node *lookup_vernode_accord_version(struct version_list_handle *handle, int64_t version)
{
struct version_list_node *vernode;
@@ -96,16 +43,49 @@ struct version_list_node *lookup_vernode_accord_version(struct version_list_hand
return NULL;
}
/*<2A><><EFBFBD><EFBFBD>ֵ:
*304-<2D>ͻ<EFBFBD><CDBB><EFBFBD><EFBFBD>Ѵﵽ<D1B4><EFB5BD><EFBFBD>°汾<C2B0><E6B1BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>consumer<65><72>׼<EFBFBD><D7BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
*300-<2D><>δ<EFBFBD><CEB4><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Client<6E><74><EFBFBD>ð汾<C3B0><E6B1BE><EFBFBD><EFBFBD>*/
static int32_t check_producer_ready_sync(struct doris_business *business, struct evhttp_request *req, int64_t cur_version)
{
const char *client_version;
int64_t clientversion;
if(NULL == (client_version=evhttp_find_header(evhttp_request_get_input_headers(req), "X-Doris-Sync-Current-Version")))
{
return HTTP_NOTMODIFIED;
}
/*request from sync client, check http posts-on-the-way first*/
if(business->posts_on_the_way)
{
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_DEBUG, "HttpProducer, posts-on-the-way: %d, meta response 300", business->posts_on_the_way);
return 300;
}
/*Client<6E>汾<EFBFBD><E6B1BE><EFBFBD><EFBFBD>? <20><><EFBFBD><EFBFBD><EFBFBD>˴ӻ<CBB4><D3BB><EFBFBD><EFBFBD>ӻ<EFBFBD><D3BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>£<EFBFBD><C2A3><EFBFBD><EFBFBD><EFBFBD>Client<6E><74><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD>Ӷ<EFBFBD><D3B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƣ<EFBFBD><C6A3>ڻ<EFBFBD><DABB><EFBFBD><EFBFBD><EFBFBD>ȡ*/
/*<2A><><EFBFBD><EFBFBD>ȡʱ<C8A1><CAB1><EFBFBD>õ<EFBFBD>300<30><30>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ȡʱ<C8A1><CAB1>ֱ<EFBFBD><D6B1>ͬ<EFBFBD><CDAC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>İ汾<C4B0><E6B1BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>汾<EFBFBD><E6B1BE><EFBFBD><EFBFBD>һ<EFBFBD>£<EFBFBD><C2A3><EFBFBD><EFBFBD><EFBFBD>304*/
if((clientversion=atol(client_version)) > cur_version)
{
business_set_sync_peer_abnormal(business);
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "HttpProducer, client version(%lu) is newer than server(%lu)", clientversion, cur_version);
return 300;
}
business_resume_sync_peer_normal(business);
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "HttpProducer, doris client is OK to sync for business: %s", business->bizname);
return HTTP_NOTMODIFIED;
}
void doris_http_server_meta_cb(struct evhttp_request *req, void *arg)
{
struct evkeyvalq params;
const char *version, *bizname;
const char *version;
int64_t verlong;
char *endptr=NULL, length[64];
struct version_list_node *vernode;
struct evbuffer *evbuf;
struct doris_business *business;
map<string, struct doris_business*>::iterator iter;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_META_REQ], 0, FS_OP_ADD, 1);
if(evhttp_parse_query(evhttp_request_get_uri(req), &params))
@@ -128,29 +108,21 @@ void doris_http_server_meta_cb(struct evhttp_request *req, void *arg)
evhttp_send_error(req, HTTP_BADREQUEST, "Parameter version invalid");
return;
}
if(NULL == (bizname = evhttp_find_header(&params, "business")))
if(NULL == (business = lookup_bizstruct_from_name(&params)))
{
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no business found");
return;
}
if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end())
{
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, business invalid");
evhttp_send_error(req, HTTP_BADREQUEST, "Parameter business invalid");
return;
}
evhttp_clear_headers(&params);
business = iter->second;
pthread_rwlock_rdlock(&business->rwlock);
if(NULL == (vernode = lookup_vernode_accord_version(business->cfgver_head, verlong)))
{
int code = check_producer_ready_sync(business, req, business->cfgver_head->latest_version);
pthread_rwlock_unlock(&business->rwlock);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_META_NONEW], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_NOTMODIFIED, "No new configs found");
evhttp_send_error(req, code, "No new configs found");
return;
}
evbuf = evbuffer_new();
@@ -210,24 +182,14 @@ struct evbuffer *evbuf_content_from_disk(struct table_list_node *tablenode, size
return evbuf;
}
void doris_response_file_range(struct evhttp_request *req, const char *bizname, const char *tablename,
int64_t verlong, size_t start, size_t end, bool range)
void doris_response_file_range(struct evhttp_request *req, struct doris_business *business,
const char *tablename, int64_t verlong, size_t start, size_t end, bool range)
{
struct version_list_node *vernode;
struct table_list_node *tablenode;
struct evbuffer *evbuf;
char length[128];
size_t filesize, res_length;
struct doris_business *business;
map<string, struct doris_business*>::iterator iter;
if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end())
{
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, business invalid");
return;
}
business = iter->second;
pthread_rwlock_rdlock(&business->rwlock);
if(NULL == (vernode = lookup_vernode_accord_version(business->cfgver_head, verlong)))
@@ -266,24 +228,29 @@ void doris_response_file_range(struct evhttp_request *req, const char *bizname,
assert(res_length == end + 1 - start);
sprintf(length, "%lu", res_length);
FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_SEND_FILE_RES], FS_OP_ADD, 1);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_FILES], 0, FS_OP_ADD, 1);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_FILE_BYTES], 0, FS_OP_ADD, res_length);
evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Length", length);
evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "application/stream");
evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "keep-alive");
if(range)
{
sprintf(length, "bytes %lu-%lu/%lu", start, end, filesize);
evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Range", length);
evhttp_send_reply(req, 206, "Partial Content", evbuf);
}
else
{
evhttp_send_reply(req, HTTP_OK, "OK", evbuf);
}
evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "application/stream");
evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "keep-alive");
evhttp_send_reply(req, HTTP_OK, "OK", evbuf);
evbuffer_free(evbuf);
}
void doris_http_server_file_cb(struct evhttp_request *req, void *arg)
{
struct evkeyvalq params;
const char *version, *tablename, *content_range, *bizname;
struct doris_business *business;
const char *version, *tablename, *content_range;
int64_t verlong;
char *endptr=NULL;
size_t req_start=0, req_end=0;
@@ -316,25 +283,22 @@ void doris_http_server_file_cb(struct evhttp_request *req, void *arg)
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_BADREQUEST, "Header Range invalid");
return;
}
if(NULL == (bizname = evhttp_find_header(&params, "business")))
}
if(NULL == (business = lookup_bizstruct_from_name(&params)))
{
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no business found");
evhttp_send_error(req, HTTP_BADREQUEST, "Parameter business invalid");
return;
}
doris_response_file_range(req, bizname, tablename, verlong, req_start, req_end, (content_range==NULL)?false:true);
doris_response_file_range(req, business, tablename, verlong, req_start, req_end, (content_range==NULL)?false:true);
evhttp_clear_headers(&params);
}
void doris_http_server_version_cb(struct evhttp_request *req, void *arg)
{
struct evkeyvalq params;
const char *bizname;
struct doris_business *business;
map<string, struct doris_business*>::iterator iter;
char verbuf[32];
if(evhttp_parse_query(evhttp_request_get_uri(req), &params))
@@ -343,23 +307,13 @@ void doris_http_server_version_cb(struct evhttp_request *req, void *arg)
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid");
return;
}
if(NULL == (bizname = evhttp_find_header(&params, "business")))
if(NULL == (business = lookup_bizstruct_from_name(&params)))
{
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no business found");
return;
}
if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end())
{
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, business invalid");
evhttp_send_error(req, HTTP_BADREQUEST, "Parameter business invalid");
return;
}
evhttp_clear_headers(&params);
business = iter->second;
pthread_rwlock_rdlock(&business->rwlock);
sprintf(verbuf, "%lu", business->cfgver_head->latest_version);
@@ -464,13 +418,6 @@ SSL_CTX *doris_connections_create_ssl_ctx(void)
return ssl_ctx;
}
struct bufferevent *doris_https_bufferevent_cb(struct event_base *evabse, void *arg)
{
SSL_CTX *ssl_instance = (SSL_CTX *)arg;
return bufferevent_openssl_socket_new(evabse, -1, SSL_new(ssl_instance), BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_CLOSE_ON_FREE);
}
void* thread_doris_http_server(void *arg)
{
struct event_base *worker_evbase;
@@ -492,9 +439,9 @@ void* thread_doris_http_server(void *arg)
evhttp_set_gencb(worker_http, doris_http_server_generic_cb, NULL);
evhttp_set_allowed_methods(worker_http, EVHTTP_REQ_GET|EVHTTP_REQ_HEAD);
if(evhttp_accept_socket(worker_http, g_doris_server_info.listener))
if(evhttp_accept_socket(worker_http, g_doris_server_info.listener_csum))
{
printf("evhttp_accept_socket %d error!\n", g_doris_server_info.listener);
printf("evhttp_accept_socket %d error!\n", g_doris_server_info.listener_csum);
assert(0); return NULL;
}