创建
This commit is contained in:
linuxrc@163.com
318
server/doris_server_scandir.cpp
Normal file
318
server/doris_server_scandir.cpp
Normal file
@@ -0,0 +1,318 @@
|
||||
#include <dirent.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include <MESA/MESA_handle_logger.h>
|
||||
|
||||
#include "doris_server_scandir.h"
|
||||
|
||||
#ifndef __FILENAME__
|
||||
#define __FILENAME__ __FILE__
|
||||
#endif
|
||||
#define MESA_RUNTIME_LOGV4(handle, lv, fmt, args...) \
|
||||
MESA_handle_runtime_log((handle), (lv), "DorisServer", "%s:%d, " fmt, __FILENAME__, __LINE__, ##args)
|
||||
|
||||
//replacement of glibc scandir, to adapt dictator malloc wrap
|
||||
#define ENLARGE_STEP 1024
|
||||
int my_scandir(const char *dir, struct dirent ***namelist,
|
||||
int(*filter)(const struct dirent *),
|
||||
int(*compar)(const void *, const void *))
|
||||
{
|
||||
DIR * od;
|
||||
int n = 0;
|
||||
int DIR_ENT_SIZE=ENLARGE_STEP;
|
||||
struct dirent ** list = NULL;
|
||||
struct dirent * p;
|
||||
struct dirent entry,*result;
|
||||
|
||||
if((dir == NULL) || (namelist == NULL) || (od = opendir(dir))==NULL)
|
||||
return -1;
|
||||
|
||||
list = (struct dirent **)malloc(DIR_ENT_SIZE*sizeof(struct dirent *));
|
||||
while(0==readdir_r(od,&entry,&result))
|
||||
{
|
||||
if(result==NULL)
|
||||
{
|
||||
break;
|
||||
}
|
||||
if( filter && !filter(&entry))
|
||||
continue;
|
||||
|
||||
p = (struct dirent *)malloc(sizeof(struct dirent));
|
||||
memcpy((void *)p,(void *)(&entry),sizeof(struct dirent));
|
||||
list[n] = p;
|
||||
|
||||
n++;
|
||||
if(n >= DIR_ENT_SIZE)
|
||||
{
|
||||
DIR_ENT_SIZE+=ENLARGE_STEP;
|
||||
list=(struct dirent **)realloc((void*)list,DIR_ENT_SIZE*sizeof(struct dirent *));
|
||||
}
|
||||
}
|
||||
closedir(od);
|
||||
*namelist = list;
|
||||
|
||||
if(compar)
|
||||
{
|
||||
qsort((void *)*namelist,n,sizeof(struct dirent *),compar);
|
||||
}
|
||||
return n;
|
||||
|
||||
}
|
||||
|
||||
int filter_fn(const struct dirent * ent)
|
||||
{
|
||||
// files in xfs are not DT_REG.
|
||||
// if(ent->d_type != DT_REG)
|
||||
// return 0;
|
||||
|
||||
return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0||
|
||||
strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0);
|
||||
}
|
||||
|
||||
enum DORIS_UPDATE_TYPE get_new_idx_path(long long current_version, const char *file_dir, void *logger, struct index_path_array **idx_path, int *idx_num)
|
||||
{
|
||||
struct dirent **namelist;
|
||||
char update_str[32]={0};
|
||||
long long latest_ful_version=0,latest_inc_version=0,config_seq=0;
|
||||
int n=0,i=0, full_file_idx=0,inc_idx_num=0;
|
||||
enum DORIS_UPDATE_TYPE update_type=CFG_UPDATE_TYPE_NONE;
|
||||
struct index_path_array *tmpidx_path_array;
|
||||
struct index_version_array *tmpidx_ver_array;
|
||||
|
||||
n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort);
|
||||
if(n < 0)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "scan dir error");
|
||||
return update_type;
|
||||
}
|
||||
|
||||
tmpidx_ver_array = (struct index_version_array*)calloc(sizeof(struct index_version_array), n);
|
||||
inc_idx_num=0;
|
||||
for(i=0;i<n;i++)
|
||||
{
|
||||
if((strcmp(namelist[i]->d_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if(strlen(namelist[i]->d_name) > 42)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "config file %s filename too long,should like full_config_index.00000000000000000001", namelist[i]->d_name);
|
||||
continue;
|
||||
}
|
||||
if(sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld",update_str,&config_seq) != 2)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "config file %s filename error,should like full_config_index.00000000000000000001", namelist[i]->d_name);
|
||||
continue;
|
||||
}
|
||||
if(strncasecmp(update_str,"full",strlen(update_str))==0)
|
||||
{
|
||||
if(config_seq>latest_ful_version)
|
||||
{
|
||||
latest_ful_version=config_seq;
|
||||
full_file_idx=i;
|
||||
}
|
||||
}
|
||||
else if(strncasecmp(update_str,"inc",strlen(update_str))==0)
|
||||
{
|
||||
if(config_seq > current_version)
|
||||
{
|
||||
tmpidx_ver_array[inc_idx_num].file_inx = i;
|
||||
tmpidx_ver_array[inc_idx_num].version = config_seq;
|
||||
inc_idx_num++;
|
||||
if(config_seq > latest_inc_version)
|
||||
{
|
||||
latest_inc_version = config_seq;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "config file %s,not full or inc config", namelist[i]->d_name);
|
||||
}
|
||||
}
|
||||
|
||||
//full update
|
||||
if(latest_ful_version > current_version)
|
||||
{
|
||||
tmpidx_path_array = (struct index_path_array *)malloc(sizeof(struct index_path_array));
|
||||
snprintf(tmpidx_path_array->path, MAX_CONFIG_FN_LEN, "%s/%s", file_dir, namelist[full_file_idx]->d_name);
|
||||
tmpidx_path_array->version = latest_ful_version;
|
||||
*idx_num = 1;
|
||||
update_type = CFG_UPDATE_TYPE_FULL;
|
||||
}
|
||||
//inc update,it's possible that do inc after full update in this function,but we'll process it at next loop.
|
||||
else if(latest_inc_version > current_version)
|
||||
{
|
||||
tmpidx_path_array = (struct index_path_array *)malloc(sizeof(struct index_path_array)*inc_idx_num);
|
||||
for(i=0;i<inc_idx_num;i++)
|
||||
{
|
||||
snprintf(tmpidx_path_array[i].path, MAX_CONFIG_FN_LEN, "%s/%s", file_dir, namelist[tmpidx_ver_array[i].file_inx]->d_name);
|
||||
tmpidx_path_array[i].version = tmpidx_ver_array[i].version;
|
||||
}
|
||||
*idx_num = inc_idx_num;
|
||||
update_type = CFG_UPDATE_TYPE_INC;
|
||||
}
|
||||
*idx_path = tmpidx_path_array;
|
||||
|
||||
free(tmpidx_ver_array);
|
||||
for(i=0;i<n;i++)
|
||||
{
|
||||
free(namelist[i]);
|
||||
}
|
||||
free(namelist);
|
||||
return update_type;
|
||||
}
|
||||
|
||||
int cm_read_cfg_index_file(const char* path, struct cfg_table_info* idx/*OUT*/, int size, void* logger)
|
||||
{
|
||||
FILE* fp=NULL;
|
||||
int ret=0,i=0;
|
||||
char line[MAX_CONFIG_LINE];
|
||||
struct stat file_info;
|
||||
|
||||
fp=fopen(path, "r");
|
||||
while(!feof(fp))
|
||||
{
|
||||
memset(line, 0, sizeof(line));
|
||||
fgets(line, sizeof(line), fp);
|
||||
ret=sscanf(line,"%s\t%d\t%s\t%s", idx[i].table_name, &(idx[i].cfg_num), idx[i].cfg_path, idx[i].encryp_algorithm);
|
||||
if((ret!=3 && ret!=4) || idx[i].cfg_num==0)//jump over empty line
|
||||
{
|
||||
continue;
|
||||
}
|
||||
ret=stat(idx[i].cfg_path, &file_info);
|
||||
if(ret!=0)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "%s of %s not exisit", idx[i].cfg_path, path);
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
idx[i].filesize = file_info.st_size;
|
||||
i++;
|
||||
if(i==size)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "Too much lines in %s", path);
|
||||
break;
|
||||
}
|
||||
}
|
||||
fclose(fp);
|
||||
return i;
|
||||
}
|
||||
|
||||
bool doris_read_table_file(struct doris_idxfile_scanner *scanner, struct cfg_table_info *table,
|
||||
struct doris_callbacks *doris_cbs, const char* dec_key, void* logger)
|
||||
{
|
||||
FILE *fp;
|
||||
size_t readlen, remainlen, oncesize;
|
||||
|
||||
if((fp = fopen(table->cfg_path, "r")) == NULL)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "fopen table file %s failed: %s", table->cfg_path, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
doris_cbs->cfgfile_start(NULL, table->table_name, table->filesize, table->cfg_num, doris_cbs->userdata);
|
||||
remainlen = table->filesize;
|
||||
while(remainlen > 0)
|
||||
{
|
||||
oncesize = (remainlen >= ONCE_BUF_SIZE)?ONCE_BUF_SIZE:remainlen;
|
||||
readlen = fread(scanner->oncebuf, 1, oncesize, fp);
|
||||
assert(readlen == oncesize);
|
||||
remainlen -= readlen;
|
||||
doris_cbs->cfgfile_update(NULL, scanner->oncebuf, readlen, doris_cbs->userdata);
|
||||
}
|
||||
doris_cbs->cfgfile_finish(NULL, doris_cbs->userdata);
|
||||
fclose(fp);
|
||||
return true;
|
||||
}
|
||||
|
||||
cJSON *doris_index_version_start(int64_t version, struct cfg_table_info *table_array, int table_num,
|
||||
enum DORIS_UPDATE_TYPE update_type, struct doris_callbacks *doris_cbs)
|
||||
{
|
||||
cJSON *meta, *array, *tmp;
|
||||
|
||||
meta = cJSON_CreateObject();
|
||||
cJSON_AddNumberToObject(meta, "type", update_type);
|
||||
cJSON_AddNumberToObject(meta, "version", version);
|
||||
|
||||
array = cJSON_CreateArray();
|
||||
for(int i=0; i<table_num; i++)
|
||||
{
|
||||
tmp = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(tmp, "tablename", table_array[i].table_name);
|
||||
cJSON_AddNumberToObject(tmp, "size", table_array[i].filesize);
|
||||
cJSON_AddNumberToObject(tmp, "cfg_num", table_array[i].cfg_num);
|
||||
cJSON_AddItemToArray(array, tmp);
|
||||
}
|
||||
cJSON_AddItemToObject(meta, "configs", array);
|
||||
|
||||
doris_cbs->version_start(NULL, meta, doris_cbs->userdata);
|
||||
return meta;
|
||||
}
|
||||
|
||||
enum DORIS_UPDATE_TYPE doris_index_file_traverse(struct doris_idxfile_scanner *scanner, const char*idx_dir,
|
||||
struct doris_callbacks *doris_cbs, const char* dec_key, void* logger)
|
||||
{
|
||||
enum DORIS_UPDATE_TYPE update_type;
|
||||
struct index_path_array *idx_path_array=NULL;
|
||||
int idx_num=0,table_num=0,i=0,j=0;
|
||||
struct cfg_table_info table_array[CM_MAX_TABLE_NUM];
|
||||
bool update_rslt;
|
||||
|
||||
memset(table_array,0,sizeof(table_array));
|
||||
update_type=get_new_idx_path(scanner->cur_version, idx_dir,logger, &idx_path_array, &idx_num);
|
||||
if(update_type == CFG_UPDATE_TYPE_NONE)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_DEBUG, "Scan over, no new configs found, waiting next.", idx_path_array[i].path);
|
||||
return update_type;
|
||||
}
|
||||
|
||||
update_rslt = true;
|
||||
for(i=0; i<idx_num; i++)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger, RLOG_LV_INFO, "load %s", idx_path_array[i].path);
|
||||
table_num=cm_read_cfg_index_file(idx_path_array[i].path, table_array, CM_MAX_TABLE_NUM, logger);
|
||||
if(table_num<0)
|
||||
{
|
||||
MESA_RUNTIME_LOGV4(logger,RLOG_LV_INFO, "load %s faild, abandon udpate.", idx_path_array[i].path);
|
||||
update_type = CFG_UPDATE_TYPE_ERR;
|
||||
break;
|
||||
}
|
||||
|
||||
cJSON *meta = doris_index_version_start(idx_path_array[i].version, table_array, table_num, update_type, doris_cbs);
|
||||
for(j=0; j<table_num && update_rslt; j++)
|
||||
{
|
||||
update_rslt = update_rslt && doris_read_table_file(scanner, table_array+j, doris_cbs, dec_key, logger);
|
||||
}
|
||||
if(update_rslt)
|
||||
{
|
||||
scanner->cur_version = idx_path_array[i].version;
|
||||
doris_cbs->version_finish(NULL, doris_cbs->userdata);
|
||||
}
|
||||
else
|
||||
{
|
||||
update_type = CFG_UPDATE_TYPE_ERR;
|
||||
doris_cbs->version_error(NULL, doris_cbs->userdata);
|
||||
cJSON_Delete(meta);
|
||||
break;
|
||||
}
|
||||
cJSON_Delete(meta);
|
||||
}
|
||||
free(idx_path_array);
|
||||
return update_type;
|
||||
}
|
||||
|
||||
struct doris_idxfile_scanner *doris_index_file_scanner(int64_t start_version)
|
||||
{
|
||||
struct doris_idxfile_scanner *scanner;
|
||||
|
||||
scanner = (struct doris_idxfile_scanner *)malloc(sizeof(struct doris_idxfile_scanner));
|
||||
scanner->cur_version = start_version;
|
||||
return scanner;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user