gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

适配非行列式透传落地文件名

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-03 10:49:52 +08:00
parent b32381f7f5
commit dcca411ddc
7 changed files with 80 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/doris_server_main.cpp
View File

@@ -17,7 +17,7 @@
#include "doris_server_http.h"
struct doris_global_info g_doris_server_info;
static unsigned long doris_vesion_20210726=20210726L;
static unsigned long doris_vesion_20210803=20210803L;
int doris_mkdir_according_path(const char * path)
{
@@ -327,7 +327,7 @@ int main(int argc, char **argv)
evhttp_set_cb(manager_http, "/doris/statistic/status", manager_statistic_status_requests_cb, NULL);
evhttp_set_cb(manager_http, "/doris/statistic/threads", manager_statistic_threads_requests_cb, NULL);
evhttp_set_gencb(manager_http, manager_generic_requests_cb, NULL);
g_doris_server_info.monitor = MESA_Monitor_instance_evhttp_new(manager_http, doris_vesion_20210726);
g_doris_server_info.monitor = MESA_Monitor_instance_evhttp_new(manager_http, doris_vesion_20210803);
if(evhttp_accept_socket(manager_http, g_doris_server_info.manager))
{
printf("evhttp_accept_socket %d error!\n", g_doris_server_info.manager);

51
server/doris_server_receive.cpp
View File

@@ -181,8 +181,8 @@ void doris_config_file_version_error(struct doris_instance *instance, void *user
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, Version %llu error, rolling back...", save->business->bizname, save->version);
}
void doris_config_file_cfgfile_start(struct doris_instance *instance, const char *tablename,
size_t size, u_int32_t cfgnum, const char *userregion, void *userdata)
void doris_config_file_cfgfile_start(struct doris_instance *instance,
const struct tablemeta *meta, const char *localpath, void *userdata)
{
struct confile_save *save=(struct confile_save *)userdata;
struct tm *localtm, savetime;
@@ -198,14 +198,14 @@ void doris_config_file_cfgfile_start(struct doris_instance *instance, const char
{
doris_mkdir_according_path(dir);
}
snprintf(save->cfg_file_path, 256, "%s/%s.%010lu", dir, tablename, save->version);
snprintf(save->cfg_file_path, 256, "%s/%s", dir, meta->filename);
if(g_doris_server_info.idx_file_maat) //MAAT<41><54>ʽ<EFBFBD><CABD>֪ͨ<CDA8>ļ<EFBFBD>
{
fprintf(save->fp_idx_file, "%s\t%u\t%s\n", tablename, cfgnum, save->cfg_file_path);
fprintf(save->fp_idx_file, "%s\t%u\t%s\n", meta->tablename, meta->cfgnum, save->cfg_file_path);
}
else //ת<><D7AA><EFBFBD><EFBFBD>ɫ<EFBFBD><C9AB><EFBFBD><EFBFBD><EFBFBD>û<EFBFBD><C3BB>Զ<EFBFBD><D4B6><EFBFBD><EFBFBD><EFBFBD>Ϣ
{
fprintf(save->fp_idx_file, "%s\t%u\t%s\t%s\n", tablename, cfgnum, save->cfg_file_path, userregion);
fprintf(save->fp_idx_file, "%s\t%u\t%s\t%s\n", meta->tablename, meta->cfgnum, save->cfg_file_path, meta->userregion);
}
if(NULL == (save->fp_cfg_file = fopen(save->cfg_file_path, "w+")))
{
@@ -306,25 +306,28 @@ void doris_config_mem_version_error(struct doris_instance *instance, void *userd
save->cur_vernode = NULL;
}
void doris_config_mem_cfgfile_start(struct doris_instance *instance, const char *tablename,
size_t size, u_int32_t cfgnum, const char *userregion, void *userdata)
void doris_config_mem_cfgfile_start(struct doris_instance *instance,
const struct tablemeta *meta, const char *localpath, void *userdata)
{
struct confile_save *save=(struct confile_save *)userdata;
save->cur_vernode->table_meta = cJSON_CreateObject();
cJSON_AddStringToObject(save->cur_vernode->table_meta, "tablename", tablename);
cJSON_AddNumberToObject(save->cur_vernode->table_meta, "cfg_num", cfgnum);
cJSON_AddNumberToObject(save->cur_vernode->table_meta, "size", size);
if(userregion != NULL)
cJSON_AddStringToObject(save->cur_vernode->table_meta, "tablename", meta->tablename);
cJSON_AddStringToObject(save->cur_vernode->table_meta, "filename", meta->filename);
cJSON_AddNumberToObject(save->cur_vernode->table_meta, "cfg_num", meta->cfgnum);
cJSON_AddNumberToObject(save->cur_vernode->table_meta, "size", meta->size);
if(meta->userregion != NULL)
{
cJSON_AddStringToObject(save->cur_vernode->table_meta, "user_region", userregion);
cJSON_AddStringToObject(save->cur_vernode->table_meta, "user_region", meta->userregion);
}
save->cur_table = (struct table_list_node *)calloc(1, sizeof(struct table_list_node));
snprintf(save->cur_table->tablename, 64, "%s", tablename);
save->cur_table->filesize = size;
save->cur_table->filesize = meta->size;
snprintf(save->cur_table->tablename, 64, "%s", meta->tablename);
snprintf(save->cur_table->localpath, 256, "%s", localpath);
TAILQ_INIT(&save->cur_table->frag_head);
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, table %s.%010llu start loading to memory...", save->business->bizname, tablename, save->version);
MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, table %s.%010llu start loading to memory...",
save->business->bizname, meta->tablename, save->version);
}
void doris_config_mem_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata)
@@ -476,13 +479,13 @@ void doris_config_localmem_version_error(struct doris_instance *instance, void *
doris_config_mem_version_error(instance, userdata);
}
}
void doris_config_localmem_cfgfile_start(struct doris_instance *instance, const char *tablename,
void doris_config_localmem_cfgfile_start(struct doris_instance *instance,
const struct tablemeta *meta, const char *localpath, void *userdata)
{
{
doris_config_common_cfgfile_start((struct confile_save *)userdata, meta->cfgnum);
if(g_doris_server_info.server_role_sw)
{
{
doris_config_mem_cfgfile_start(instance, meta, localpath, userdata);
}
}
@@ -549,19 +552,19 @@ void doris_config_version_error(struct doris_instance *instance, void *userdata)
doris_config_mem_version_error(instance, userdata);
}
}
void doris_config_cfgfile_start(struct doris_instance *instance, const char *tablename,
void doris_config_cfgfile_start(struct doris_instance *instance,
const struct tablemeta *meta, const char *localpath, void *userdata)
{
struct confile_save *save=(struct confile_save *)userdata;
doris_config_common_cfgfile_start((struct confile_save *)userdata, meta->cfgnum);
if(save->business->write_file_sw)
{
{
doris_config_file_cfgfile_start(instance, meta, localpath, userdata);
}
if(g_doris_server_info.server_role_sw)
{
{
doris_config_mem_cfgfile_start(instance, meta, save->cfg_file_path, userdata);
}
}

1
server/doris_server_receive.h
View File

@@ -58,6 +58,7 @@ struct cont_frag_node
struct table_list_node
{
char tablename[64];
char localpath[256];
size_t filesize;
size_t cur_totallen;

16
server/doris_server_scandir.cpp
View File

@@ -231,7 +231,8 @@ bool doris_read_table_file(struct doris_idxfile_scanner *scanner, struct cfg_tab
FILE *fp;
size_t readlen, remainlen, oncesize;
MD5_CTX md5ctx;
char md5buffer[64], *user_region=NULL;
char md5buffer[64], *filename;
struct tablemeta meta;
if((fp = fopen(table->cfg_path, "r")) == NULL)
{
@@ -240,11 +241,14 @@ bool doris_read_table_file(struct doris_idxfile_scanner *scanner, struct cfg_tab
}
MD5_Init(&md5ctx);
if(table->user_region[0] != '\0')
{
user_region = table->user_region;
}
doris_cbs->cfgfile_start(NULL, table->table_name, table->filesize, table->cfg_num, user_region, doris_cbs->userdata);
meta.tablename = table->table_name;
filename = strrchr(table->cfg_path, '/');
meta.filename = (filename!=NULL)?(filename + 1):table->cfg_path;
meta.userregion = (table->user_region[0] != '\0')?table->user_region:NULL;
meta.size = table->filesize;
meta.cfgnum = table->cfg_num;
doris_cbs->cfgfile_start(NULL, &meta, table->cfg_path, doris_cbs->userdata);
remainlen = table->filesize;
while(remainlen > 0)
{