gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

带全量的业务，支持持久化的旧配置淘汰删除；

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-30 14:22:48 +08:00
parent bc525d48e4
commit 6386e5de57
6 changed files with 176 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
server/doris_server_receive.cpp
View File

@@ -154,14 +154,14 @@ void doris_config_file_version_start(struct doris_csum_instance *instance, cJSON
if(business->type == CFG_UPDATE_TYPE_FULL)
{
snprintf(business->inc_index_path, 512, "%s/inc/index/full_config_index.%010lu", business->store_path_root, business->version);
snprintf(business->tmp_index_path, 512, "%s/inc/full_config_index.%010lu.ing", business->store_path_root, business->version);
snprintf(business->full_index_path, 512, "%s/full/index/full_config_index.%010lu", business->store_path_root, business->version);
snprintf(business->inc_index_path, 256, "%s/inc/index/full_config_index.%010lu", business->store_path_root, business->version);
snprintf(business->tmp_index_path, 256, "%s/inc/full_config_index.%010lu.ing", business->store_path_root, business->version);
snprintf(business->full_index_path, 256, "%s/full/index/full_config_index.%010lu", business->store_path_root, business->version);
}
else
{
snprintf(business->inc_index_path, 512, "%s/inc/index/inc_config_index.%010lu", business->store_path_root, business->version);
snprintf(business->tmp_index_path, 512, "%s/inc/full_config_index.%010lu.ing", business->store_path_root, business->version);
snprintf(business->inc_index_path, 256, "%s/inc/index/inc_config_index.%010lu", business->store_path_root, business->version);
snprintf(business->tmp_index_path, 256, "%s/inc/full_config_index.%010lu.ing", business->store_path_root, business->version);
}
if(NULL==(business->fp_idx_file = fopen(business->tmp_index_path, "w+")))
{
@@ -173,6 +173,7 @@ void doris_config_file_version_start(struct doris_csum_instance *instance, cJSON
void doris_config_file_version_finish(struct doris_csum_instance *instance, void *userdata)
{
struct doris_business *business=(struct doris_business *)userdata;
char tmp_index_dir[256];
fclose(business->fp_idx_file);
if(rename(business->tmp_index_path, business->inc_index_path))
@@ -189,6 +190,19 @@ void doris_config_file_version_finish(struct doris_csum_instance *instance, void
business->bizname, business->tmp_index_path, business->inc_index_path, strerror(errno));
assert(0);
}
if(business->saves_when_fulldel > 0)
{
for(u_int32_t i=1; i<business->saves_when_fulldel; i++)
{
business->full_version_inc[i-1] = business->full_version_inc[i]; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>򣬽<EFBFBD><F2A3ACBD>µİ汾<C4B0><E6B1BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵ
}
business->full_version_inc[business->saves_when_fulldel-1] = business->version;
snprintf(tmp_index_dir, 256, "%s/full/index", business->store_path_root);
remove_configs_version_smaller(tmp_index_dir, business->full_version_inc[0], 0, g_doris_server_info.log_runtime);
snprintf(tmp_index_dir, 256, "%s/inc/index", business->store_path_root);
remove_configs_version_smaller(tmp_index_dir, business->full_version_inc[0], 1, g_doris_server_info.log_runtime);
}
}
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, Version %lu write finished, index file: %s",
business->bizname, business->version, business->inc_index_path);
@@ -655,6 +669,10 @@ void* thread_doris_client_recv_cfg(void *arg)
doris_cbs.userdata = business;
snprintf(stored_path, 512, "%s/full/index", business->store_path_root);
if(business->saves_when_fulldel > 0)
{
get_full_topN_max_versions(stored_path, business->full_version_inc, business->saves_when_fulldel);
}
update_type = doris_index_file_traverse(scanner, stored_path, &doris_cbs, NULL, g_doris_server_info.log_runtime);
snprintf(stored_path, 512, "%s/inc/index", business->store_path_root);
do {
@@ -735,6 +753,10 @@ void* thread_index_file_recv_cfg(void *arg)
snprintf(stored_path, 512, "%s/full/index", business->store_path_root);
update_type = doris_index_file_traverse(timer_priv.scanner, stored_path, &timer_priv.doris_cbs, NULL, g_doris_server_info.log_runtime);
if(business->saves_when_fulldel > 0)
{
get_full_topN_max_versions(stored_path, business->full_version_inc, business->saves_when_fulldel);
}
snprintf(stored_path, 512, "%s/inc/index", business->store_path_root);
do{
update_type = doris_index_file_traverse(timer_priv.scanner, stored_path, &timer_priv.doris_cbs, NULL, g_doris_server_info.log_runtime);
@@ -1002,7 +1024,7 @@ void http_config_direct_version_cancel(struct version_list_node *vernode, struct
remove(tablenode->localpath);
}
config_version_node_cleanup(vernode);
if(evtimer_pending(&vernode->timer_expire, NULL))
if(business->concurrency_allowed && evtimer_pending(&vernode->timer_expire, NULL))
{
evtimer_del(&vernode->timer_expire);
}
@@ -1096,12 +1118,12 @@ void doris_config_post_version_finish(struct doris_business *business, struct ve
if(vernode->cfg_type == CFG_UPDATE_TYPE_FULL)
{
snprintf(business->inc_index_path, 512, "%s/inc/index/full_config_index.%010lu", business->store_path_root, vernode->version);
snprintf(business->full_index_path, 512, "%s/full/index/full_config_index.%010lu", business->store_path_root, vernode->version);
snprintf(business->inc_index_path, 256, "%s/inc/index/full_config_index.%010lu", business->store_path_root, vernode->version);
snprintf(business->full_index_path, 256, "%s/full/index/full_config_index.%010lu", business->store_path_root, vernode->version);
}
else
{
snprintf(business->inc_index_path, 512, "%s/inc/index/inc_config_index.%010lu", business->store_path_root, vernode->version);
snprintf(business->inc_index_path, 256, "%s/inc/index/inc_config_index.%010lu", business->store_path_root, vernode->version);
}
/*HTTP postʱ<74><CAB1><EFBFBD><EFBFBD><EFBFBD>汾<EFBFBD><E6B1BE><EFBFBD><EFBFBD>ÿ<EFBFBD><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Լ<EFBFBD><D4BC><EFBFBD><EFBFBD><EFBFBD>ʱ֪ͨ<CDA8>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ñ<EFBFBD><C3B1><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC>Ĺرպ<D8B1><D5BA><EFBFBD>*/
sprintf(business->tmp_index_path, "%s", vernode->tmp_index_path);
@@ -1139,7 +1161,7 @@ void http_config_direct_version_finish(struct version_list_node *vernode, struct
char version[32], token[64];
int64_t new_version;
if(evtimer_pending(&vernode->timer_expire, NULL))
if(business->concurrency_allowed && evtimer_pending(&vernode->timer_expire, NULL))
{
evtimer_del(&vernode->timer_expire);
}
@@ -1220,7 +1242,7 @@ void http_prod_server_verion_end_cb(struct evhttp_request *req, void *arg)
{
sprintf(version, "%lu", vernode->version);
evhttp_add_header(evhttp_request_get_output_headers(req), "X-Set-Version", version);
evhttp_send_error(req, HTTP_OK, "version already finished"); //<2F><>֤<EFBFBD><D6A4><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD>
evhttp_send_reply(req, HTTP_OK, "version already finished", NULL); //<2F><>֤<EFBFBD><D6A4><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD>
return;
}
if(vernode->cur_table != NULL || vernode->syncing)
@@ -1312,11 +1334,11 @@ void doris_config_post_version_start(struct version_list_node *cur_vernode, cons
snprintf(cur_vernode->token, 64, "%s", token);
if(cur_vernode->cfg_type == CFG_UPDATE_TYPE_FULL)
{
snprintf(cur_vernode->tmp_index_path, 512, "%s/inc/full_config_index.%s.ing", business->store_path_root, token);
snprintf(cur_vernode->tmp_index_path, 256, "%s/inc/full_config_index.%s.ing", business->store_path_root, token);
}
else
{
snprintf(cur_vernode->tmp_index_path, 512, "%s/inc/full_config_index.%s.ing", business->store_path_root, token);
snprintf(cur_vernode->tmp_index_path, 256, "%s/inc/full_config_index.%s.ing", business->store_path_root, token);
}
if(NULL==(cur_vernode->fp_idx_file = fopen(cur_vernode->tmp_index_path, "w+")))
{
@@ -2105,6 +2127,10 @@ void* thread_http_post_recv_cfg(void *arg)
doris_cbs.userdata = business;
snprintf(stored_path, 512, "%s/full/index", business->store_path_root);
if(business->saves_when_fulldel > 0)
{
get_full_topN_max_versions(stored_path, business->full_version_inc, business->saves_when_fulldel);
}
update_type = doris_index_file_traverse(scanner, stored_path, &doris_cbs, NULL, g_doris_server_info.log_runtime);
snprintf(stored_path, 512, "%s/inc/index", business->store_path_root);
do {