gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

限制缓存在内存的配置文件内容的版本数量，以适应全增量模式的配置下发

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-04 11:14:56 +08:00
parent dcca411ddc
commit 730a744229
9 changed files with 160 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
server/doris_server_http.cpp
View File

@@ -12,6 +12,9 @@
#include <errno.h>
#include <sys/prctl.h>
#include <poll.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <event2/bufferevent_ssl.h>
@@ -71,6 +74,28 @@ int doris_create_listen_socket(int bind_port)
return listener;
}
struct version_list_node *lookup_vernode_accord_version(struct version_list_handle *handle, int64_t version)
{
struct version_list_node *vernode;
map<int64_t, struct version_list_node*>::iterator iter;
vernode = TAILQ_FIRST(&handle->version_head);
if(vernode!=NULL && version<vernode->version) //<2F><>ǰȫ<C7B0><C8AB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>С<EFBFBD>İ汾<C4B0><E6B1BE>
{
version = vernode->version;
}
for(; version <= handle->latest_version; version++)
{
if((iter = handle->version2node->find(version)) != handle->version2node->end())
{
vernode = iter->second;
return vernode;
}
}
return NULL;
}
void doris_http_server_meta_cb(struct evhttp_request *req, void *arg)
{
struct evkeyvalq params;
@@ -121,19 +146,13 @@ void doris_http_server_meta_cb(struct evhttp_request *req, void *arg)
business = iter->second;
pthread_rwlock_rdlock(&business->rwlock);
if(verlong > business->cfgver_head->latest_version)
if(NULL == (vernode = lookup_vernode_accord_version(business->cfgver_head, verlong)))
{
pthread_rwlock_unlock(&business->rwlock);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_META_NONEW], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_NOTMODIFIED, "No new configs found");
return;
}
vernode = TAILQ_FIRST(&business->cfgver_head->version_head);
while(vernode->version < verlong)
{
vernode = TAILQ_NEXT(vernode, version_node);
}
evbuf = evbuffer_new();
evbuffer_add(evbuf, vernode->metacont, vernode->metalen);
sprintf(length, "%u", vernode->metalen);
@@ -147,15 +166,58 @@ void doris_http_server_meta_cb(struct evhttp_request *req, void *arg)
evbuffer_free(evbuf);
}
struct evbuffer *evbuf_content_from_memory(struct table_list_node *tablenode, size_t start, size_t end, size_t *length)
{
struct evbuffer *evbuf;
struct cont_frag_node *fragnode;
size_t copy_len, offset=start, res_length=0;
evbuf = evbuffer_new();
for(fragnode=TAILQ_FIRST(&tablenode->frag_head); fragnode!=NULL && fragnode->start<=end; fragnode=TAILQ_NEXT(fragnode, frag_node))
{
if(offset > fragnode->end)
{
continue;
}
copy_len = (end>fragnode->end)?(fragnode->end-offset + 1):(end-offset + 1);
evbuffer_add(evbuf, fragnode->content+(offset-fragnode->start), copy_len);
offset += copy_len;
res_length += copy_len;
}
*length = res_length;
return evbuf;
}
struct evbuffer *evbuf_content_from_disk(struct table_list_node *tablenode, size_t start, size_t end, size_t *length)
{
struct evbuffer *evbuf;
int32_t fd;
if((fd = open(tablenode->localpath, O_RDONLY, 0)) < 0)
{
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Send response, open %s failed: %s", strerror(errno));
return NULL;
}
evbuf = evbuffer_new();
if(evbuffer_add_file(evbuf, fd, start, end-start+1))
{
evbuffer_free(evbuf);
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Send response, evbuffer_add_file %s failed");
close(fd);
return NULL;
}
*length = evbuffer_get_length(evbuf);
return evbuf;
}
void doris_response_file_range(struct evhttp_request *req, const char *bizname, const char *tablename,
int64_t verlong, size_t start, size_t end, bool range)
{
struct version_list_node *vernode;
struct table_list_node *tablenode;
struct cont_frag_node *fragnode;
struct evbuffer *evbuf;
char length[128];
size_t filesize, res_length=0, copy_len, offset=start;
size_t filesize, res_length;
struct doris_business *business;
map<string, struct doris_business*>::iterator iter;
@@ -168,18 +230,13 @@ void doris_response_file_range(struct evhttp_request *req, const char *bizname,
business = iter->second;
pthread_rwlock_rdlock(&business->rwlock);
if(verlong > business->cfgver_head->latest_version)
if(NULL == (vernode = lookup_vernode_accord_version(business->cfgver_head, verlong)))
{
pthread_rwlock_unlock(&business->rwlock);
FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_FILE_RES_404], 0, FS_OP_ADD, 1);
evhttp_send_error(req, HTTP_NOTFOUND, "Version too old");
return;
}
vernode = TAILQ_FIRST(&business->cfgver_head->version_head);
while(vernode->version < verlong)
{
vernode = TAILQ_NEXT(vernode, version_node);
}
tablenode = TAILQ_FIRST(&vernode->table_head);
while(tablenode!=NULL && strcmp(tablename, tablenode->tablename))
{
@@ -197,17 +254,13 @@ void doris_response_file_range(struct evhttp_request *req, const char *bizname,
{
end = tablenode->filesize - 1;
}
evbuf = evbuffer_new();
for(fragnode=TAILQ_FIRST(&tablenode->frag_head); fragnode!=NULL && fragnode->start<=end; fragnode=TAILQ_NEXT(fragnode, frag_node))
if(vernode->cont_in_disk)
{
if(offset > fragnode->end)
{
continue;
}
copy_len = (end>fragnode->end)?(fragnode->end-offset + 1):(end-offset + 1);
evbuffer_add(evbuf, fragnode->content+(offset-fragnode->start), copy_len);
offset += copy_len;
res_length += copy_len;
evbuf = evbuf_content_from_disk(tablenode, start, end, &res_length);
}
else
{
evbuf = evbuf_content_from_memory(tablenode, start, end, &res_length);
}
pthread_rwlock_unlock(&business->rwlock);
@@ -390,11 +443,6 @@ void* thread_doris_http_server(void *arg)
if(g_doris_server_info.ssl_conn_on)
{
g_doris_server_info.ssl_instance = doris_connections_create_ssl_ctx();
if(g_doris_server_info.ssl_instance == NULL)
{
assert(0);return NULL;
}
evhttp_set_bevcb(worker_http, doris_https_bufferevent_cb, g_doris_server_info.ssl_instance);
}