gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持多业务系统的配置并发进行同步

Browse Source
This commit is contained in:
linuxrc@163.com
2021-07-22 10:25:42 +08:00
parent 8b236b38cb
commit 5217e9188e
10 changed files with 453 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
server/doris_server_http.cpp
View File

@@ -73,47 +73,63 @@ int doris_create_listen_socket(int bind_port)
void doris_http_server_meta_cb(struct evhttp_request *req, void *arg)
{
struct worker_statistic_info *statistic=(struct worker_statistic_info *)arg;
struct evkeyvalq params;
const char *version;
const char *version, *bizname;
int64_t verlong;
char *endptr=NULL, length[64];
struct version_list_node *vernode;
struct evbuffer *evbuf;
struct doris_business *business;
map<string, struct doris_business*>::iterator iter;
statistic->statistic.field[DRS_FSSTAT_CLIENT_META_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_META_REQ], 0, FS_OP_ADD, 1);
if(evhttp_parse_query(evhttp_request_get_uri(req), &params))
{
statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid");
return;
}
if(NULL == (version = evhttp_find_header(&params, "version")))
{
statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no version found");
return;
}
if(0==(verlong = strtol(version, &endptr, 10)) || *endptr!='\0')
{
statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameter version invalid");
return;
}
evhttp_clear_headers(&params);
pthread_rwlock_rdlock(&g_doris_server_info.rwlock);
if(verlong > g_doris_server_info.cfgver_head->latest_version)
if(NULL == (bizname = evhttp_find_header(&params, "business")))
{
pthread_rwlock_unlock(&g_doris_server_info.rwlock);
statistic->statistic.field[DRS_FSSTAT_SEND_META_NONEW] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no business found");
return;
}
if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end())
{
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, business invalid");
return;
}
evhttp_clear_headers(&params);
business = iter->second;
pthread_rwlock_rdlock(&business->rwlock);
if(verlong > business->cfgver_head->latest_version)
{
pthread_rwlock_unlock(&business->rwlock);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_META_NONEW], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_NOTMODIFIED, "No new configs found");
return;
}
vernode = TAILQ_FIRST(&g_doris_server_info.cfgver_head->version_head);
vernode = TAILQ_FIRST(&business->cfgver_head->version_head);
while(vernode->version < verlong)
{
vernode = TAILQ_NEXT(vernode, version_node);
@@ -121,9 +137,9 @@ void doris_http_server_meta_cb(struct evhttp_request *req, void *arg)
evbuf = evbuffer_new();
evbuffer_add(evbuf, vernode->metacont, vernode->metalen);
sprintf(length, "%u", vernode->metalen);
pthread_rwlock_unlock(&g_doris_server_info.rwlock);
pthread_rwlock_unlock(&business->rwlock);
statistic->statistic.field[DRS_FSSTAT_SEND_META_RES] += 1;
FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_SEND_META_RES], FS_OP_ADD, 1);
evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "application/json");
evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "keep-alive");
evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Length", length);
@@ -131,8 +147,8 @@ void doris_http_server_meta_cb(struct evhttp_request *req, void *arg)
evbuffer_free(evbuf);
}
void doris_response_file_range(struct evhttp_request *req, const char *tablename,
int64_t verlong, size_t start, size_t end, bool range, struct worker_statistic_info *statistic)
void doris_response_file_range(struct evhttp_request *req, const char *bizname, const char *tablename,
int64_t verlong, size_t start, size_t end, bool range)
{
struct version_list_node *vernode;
struct table_list_node *tablenode;
@@ -140,16 +156,26 @@ void doris_response_file_range(struct evhttp_request *req, const char *tablename
struct evbuffer *evbuf;
char length[128];
size_t filesize, res_length=0, copy_len, offset=start;
struct doris_business *business;
map<string, struct doris_business*>::iterator iter;
pthread_rwlock_rdlock(&g_doris_server_info.rwlock);
if(verlong > g_doris_server_info.cfgver_head->latest_version)
if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end())
{
pthread_rwlock_unlock(&g_doris_server_info.rwlock);
statistic->statistic.field[DRS_FSSTAT_SEND_FILE_RES_404] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, business invalid");
return;
}
business = iter->second;
pthread_rwlock_rdlock(&business->rwlock);
if(verlong > business->cfgver_head->latest_version)
{
pthread_rwlock_unlock(&business->rwlock);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_FILE_RES_404], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_NOTFOUND, "Version too old");
return;
}
vernode = TAILQ_FIRST(&g_doris_server_info.cfgver_head->version_head);
vernode = TAILQ_FIRST(&business->cfgver_head->version_head);
while(vernode->version < verlong)
{
vernode = TAILQ_NEXT(vernode, version_node);
@@ -161,8 +187,8 @@ void doris_response_file_range(struct evhttp_request *req, const char *tablename
}
if(tablenode==NULL || start>tablenode->filesize)
{
pthread_rwlock_unlock(&g_doris_server_info.rwlock);
statistic->statistic.field[DRS_FSSTAT_SEND_FILE_RES_404] += 1;
pthread_rwlock_unlock(&business->rwlock);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_FILE_RES_404], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_NOTFOUND, "No valid content found");
return;
}
@@ -183,12 +209,12 @@ void doris_response_file_range(struct evhttp_request *req, const char *tablename
offset += copy_len;
res_length += copy_len;
}
pthread_rwlock_unlock(&g_doris_server_info.rwlock);
pthread_rwlock_unlock(&business->rwlock);
assert(res_length == end + 1 - start);
sprintf(length, "%lu", res_length);
statistic->statistic.field[DRS_FSSTAT_SEND_FILE_RES] += 1;
statistic->statistic.field[DRS_FSSTAT_SEND_FILE_BYTES] += res_length;
FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_SEND_FILE_RES], FS_OP_ADD, 1);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_FILE_BYTES], 0, FS_OP_ADD, res_length);
evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Length", length);
if(range)
{
@@ -203,31 +229,30 @@ void doris_response_file_range(struct evhttp_request *req, const char *tablename
void doris_http_server_file_cb(struct evhttp_request *req, void *arg)
{
struct worker_statistic_info *statistic=(struct worker_statistic_info *)arg;
struct evkeyvalq params;
const char *version, *tablename, *content_range;
const char *version, *tablename, *content_range, *bizname;
int64_t verlong;
char *endptr=NULL;
size_t req_start=0, req_end=0;
statistic->statistic.field[DRS_FSSTAT_CLIENT_FILE_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_FILE_REQ], 0, FS_OP_ADD, 1);
if(evhttp_parse_query(evhttp_request_get_uri(req), &params))
{
statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid");
return;
}
if(NULL==(version=evhttp_find_header(&params, "version")) || NULL==(tablename=evhttp_find_header(&params, "tablename")))
{
evhttp_clear_headers(&params);
statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no version/tablename found");
return;
}
if(0==(verlong = strtol(version, &endptr, 10)) || *endptr!='\0')
{
evhttp_clear_headers(&params);
statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameter version invalid");
return;
}
@@ -235,12 +260,19 @@ void doris_http_server_file_cb(struct evhttp_request *req, void *arg)
sscanf(content_range, "%*[^0-9]%lu-%lu", &req_start, &req_end)<1)
{
evhttp_clear_headers(&params);
statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1;
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_BADREQUEST, "Header Range invalid");
return;
}
if(NULL == (bizname = evhttp_find_header(&params, "business")))
{
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1);
evhttp_clear_headers(&params);
evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no business found");
return;
}
doris_response_file_range(req, tablename, verlong, req_start, req_end, (content_range==NULL)?false:true, statistic);
doris_response_file_range(req, bizname, tablename, verlong, req_start, req_end, (content_range==NULL)?false:true);
evhttp_clear_headers(&params);
}
@@ -350,14 +382,10 @@ void* thread_doris_http_server(void *arg)
{
struct event_base *worker_evbase;
struct evhttp *worker_http;
struct worker_statistic_info statistic;
struct timeval tv;
prctl(PR_SET_NAME, "http_server");
memset(&statistic, 0, sizeof(struct worker_statistic_info));
worker_evbase = event_base_new();
worker_http = evhttp_new(worker_evbase);
if(g_doris_server_info.ssl_conn_on)
@@ -370,9 +398,9 @@ void* thread_doris_http_server(void *arg)
evhttp_set_bevcb(worker_http, doris_https_bufferevent_cb, g_doris_server_info.ssl_instance);
}
evhttp_set_cb(worker_http, "/configmeta", doris_http_server_meta_cb, &statistic);
evhttp_set_cb(worker_http, "/configfile", doris_http_server_file_cb, &statistic);
evhttp_set_gencb(worker_http, doris_http_server_generic_cb, &statistic);
evhttp_set_cb(worker_http, "/configmeta", doris_http_server_meta_cb, NULL);
evhttp_set_cb(worker_http, "/configfile", doris_http_server_file_cb, NULL);
evhttp_set_gencb(worker_http, doris_http_server_generic_cb, NULL);
evhttp_set_allowed_methods(worker_http, EVHTTP_REQ_GET|EVHTTP_REQ_HEAD);
if(evhttp_accept_socket(worker_http, g_doris_server_info.listener))
@@ -381,11 +409,6 @@ void* thread_doris_http_server(void *arg)
assert(0); return NULL;
}
evtimer_assign(&statistic.timer_statistic, worker_evbase, doris_worker_statistic_timer_cb, &statistic);
tv.tv_sec = g_doris_server_info.fsstat_period;
tv.tv_usec = 0;
evtimer_add(&statistic.timer_statistic, &tv);
event_base_dispatch(worker_evbase);
printf("Libevent dispath error, should not run here.\n");
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Libevent dispath error, should not run here.");