gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加HTTP Post上传配置接口，支持主从双机备份与同步（冷备）

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-25 18:40:20 +08:00
parent 67bafbefc9
commit 1aca701f12
28 changed files with 3295 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
client/doris_client_fetch.cpp
View File

@@ -35,29 +35,6 @@ static int doris_md5_final_string(MD5_CTX *c, char *result, unsigned int size)
return 0;
}
void easy_string_destroy(struct easy_string *estr)
{
if(estr->buff != NULL)
{
free(estr->buff);
estr->buff = NULL;
estr->len = estr->size = 0;
}
}
void easy_string_savedata(struct easy_string *estr, const char *data, size_t len)
{
if(estr->size-estr->len < len+1)
{
estr->size += len*4+1;
estr->buff = (char*)realloc(estr->buff, estr->size);
}
memcpy(estr->buff+estr->len, data, len);
estr->len += len;
estr->buff[estr->len]='\0';
}
void doris_confile_ctx_reset(struct doris_confile_ctx *ctx)
{
//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֲ<EFBFBD><D6B2><EFBFBD>
@@ -75,12 +52,12 @@ void doris_confile_ctx_destry(struct doris_confile_ctx *ctx)
ctx->httpctx = NULL;
}
void doris_update_new_version(struct doris_instance *instance)
void doris_update_new_version(struct doris_csum_instance *instance)
{
instance->cur_version = instance->new_version;
}
void doris_request_restart_timer(struct doris_instance *instance, time_t wait_s)
void doris_request_restart_timer(struct doris_csum_instance *instance, time_t wait_s)
{
struct timeval tv;
@@ -89,7 +66,7 @@ void doris_request_restart_timer(struct doris_instance *instance, time_t wait_s)
event_add(&instance->timer_fetch, &tv);
}
void doris_fetch_next_confile_meta(struct doris_instance *instance)
void doris_fetch_next_confile_meta(struct doris_csum_instance *instance)
{
cJSON *cur_a_item, *sub;
@@ -125,7 +102,7 @@ void doris_fetch_next_confile_meta(struct doris_instance *instance)
void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code, long res_code, void *userp)
{
struct doris_instance *instance = (struct doris_instance *)userp;
struct doris_csum_instance *instance = (struct doris_csum_instance *)userp;
const char *pos_colon;
size_t datalen;
char buffer[64];
@@ -180,7 +157,7 @@ void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code
void doris_http_confile_body_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp)
{
struct doris_instance *instance = (struct doris_instance *)userp;
struct doris_csum_instance *instance = (struct doris_csum_instance *)userp;
if(code!=CURLE_OK || (instance->ctx.res_code!=200 && instance->ctx.res_code!=206) || (res_code!=200 && res_code!=206))
{
@@ -193,10 +170,10 @@ void doris_http_confile_body_cb(const char *ptr, size_t bytes, CURLcode code, lo
instance->statistic.field[DRS_FS_FILED_RES_BYTES] += bytes;
}
void doris_http_fetch_confile(struct doris_instance *instance);
void doris_http_fetch_confile(struct doris_csum_instance *instance);
void doris_http_confile_done_cb(CURLcode res, long res_code, const char *err, void *userp)
{
struct doris_instance *instance = (struct doris_instance *)userp;
struct doris_csum_instance *instance = (struct doris_csum_instance *)userp;
char md5buffer[64];
bool direct_fail=false;
@@ -284,7 +261,7 @@ out_md5:
doris_request_restart_timer(instance, instance->param->retry_interval);
}
void doris_http_fetch_confile(struct doris_instance *instance)
void doris_http_fetch_confile(struct doris_csum_instance *instance)
{
struct doris_http_callback curlcbs;
char metauri[128], range[64]={0};
@@ -319,7 +296,7 @@ void doris_http_fetch_confile(struct doris_instance *instance)
void doris_http_meta_header_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp)
{
struct doris_instance *instance = (struct doris_instance *)userp;
struct doris_csum_instance *instance = (struct doris_csum_instance *)userp;
//check code only once
if(instance->ctx.res_code != 0)
@@ -331,14 +308,14 @@ void doris_http_meta_header_cb(const char *ptr, size_t bytes, CURLcode code, lon
if(res_code != 200)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "No new meta found, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d",
instance->ctx.server, instance->cur_version, instance->req_version, code);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "business: %s, No new meta found, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d",
instance->args.bizname, instance->ctx.server, instance->cur_version, instance->req_version, code);
}
}
void doris_http_meta_body_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp)
{
struct doris_instance *instance = (struct doris_instance *)userp;
struct doris_csum_instance *instance = (struct doris_csum_instance *)userp;
if(code!=CURLE_OK || res_code!=200)
{
@@ -350,14 +327,14 @@ void doris_http_meta_body_cb(const char *ptr, size_t bytes, CURLcode code, long
void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void *userp)
{
struct doris_instance *instance = (struct doris_instance *)userp;
struct doris_csum_instance *instance = (struct doris_csum_instance *)userp;
cJSON *sub;
int64_t new_version;
if(res!=CURLE_OK)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Request meta failed, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s",
instance->ctx.server, instance->cur_version, instance->req_version, res_code, err);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "business: %s, Request meta failed, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s",
instance->args.bizname, instance->ctx.server, instance->cur_version, instance->req_version, res_code, err);
goto out_error;
}
@@ -369,16 +346,16 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void
instance->meta = cJSON_Parse(instance->estr.buff);
if(instance->meta == NULL)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Parse meta failed, server: %s, req_version=%lu, invalid json: %s",
instance->ctx.server, instance->req_version, instance->estr.buff);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "business: %s, Parse meta failed, server: %s, req_version=%lu, invalid json: %s",
instance->args.bizname, instance->ctx.server, instance->req_version, instance->estr.buff);
goto out_error;
}
sub = cJSON_GetObjectItem(instance->meta, "version");
new_version = sub->valuedouble;
if(new_version <= instance->cur_version)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "An older version received, abandon it. server: %s, cur_version=%lu, invalid json: %s",
instance->ctx.server, instance->cur_version, instance->estr.buff);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "business: %s, An older version received, abandon it. server: %s, cur_version=%lu, invalid json: %s",
instance->args.bizname, instance->ctx.server, instance->cur_version, instance->estr.buff);
cJSON_Delete(instance->meta);
instance->meta = NULL;
goto out_error;
@@ -386,8 +363,8 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void
instance->new_version = new_version;
instance->req_version = instance->new_version;
instance->statistic.field[DRS_FS_FILED_RES_META] += 1;
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, server: %s, cur_version=%lu, newjson: %s",
instance->ctx.server, instance->cur_version, instance->estr.buff);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "business: %s, NEW_META found, server: %s, cur_version=%lu, newjson: %s",
instance->args.bizname, instance->ctx.server, instance->cur_version, instance->estr.buff);
instance->cbs.version_start(instance, instance->meta, instance->cbs.userdata);
instance->array = cJSON_GetObjectItem(instance->meta, "configs");
@@ -405,13 +382,21 @@ out_error:
doris_request_restart_timer(instance, instance->param->retry_interval);
easy_string_destroy(&instance->estr);
doris_confile_ctx_destry(&instance->ctx);
if(res_code==304 && instance->cbs.version_updated!=NULL) //<2F>汾<EFBFBD><E6B1BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC>
{
instance->cbs.version_updated(instance, instance->cbs.userdata);
}
if(res_code==300 && instance->param->client_sync_on) //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>а<EFBFBD>;<EFBFBD>еİ汾<C4B0>ϴ<EFBFBD>
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "business: %s, Server is busy processing version requests, waiting it done...", instance->args.bizname);
}
}
static void doris_http_fetch_meta(struct doris_instance *instance)
static void doris_http_fetch_meta(struct doris_csum_instance *instance)
{
u_int64_t balance_seed;
struct doris_http_callback curlcbs;
char metauri[128];
char metauri[128], cur_version[128];
balance_seed = (((u_int64_t)rand()&0xFFFF) << 48) | (((u_int64_t)rand()&0xFFFF) << 32) |
(((u_int64_t)rand()&0xFFFF) << 16) | ((u_int64_t)rand()&0xFFFF);
@@ -439,6 +424,13 @@ static void doris_http_fetch_meta(struct doris_instance *instance)
instance->req_version = instance->cur_version + 1; //ֻ<>а汾<D0B0><E6B1BE><EFBFBD>³ɹ<C2B3><C9B9><EFBFBD><EFBFBD><EFBFBD>cur_version<6F>Ż<EFBFBD><C5BB><EFBFBD><EFBFBD><EFBFBD>
if(instance->ctx.httpctx != NULL)
{
if(instance->param->client_sync_on)
{
//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC><EFBFBD><EFBFBD><EFBFBD>ѣ<EFBFBD><D1A3><EFBFBD><EFBFBD>ڱ<EFBFBD><DAB1><EFBFBD><EFBFBD>Ǵ<EFBFBD>Client<6E><74><EFBFBD><EFBFBD>
sprintf(cur_version, "X-Doris-Sync-Current-Version: %lu", instance->cur_version);
doris_http_ctx_add_header(instance->ctx.httpctx, cur_version);
}
snprintf(metauri, 128, "configmeta?version=%lu&business=%s", instance->req_version, instance->args.bizname);
if(!doris_http_launch_get_request(instance->ctx.httpctx, metauri))
{
@@ -464,7 +456,7 @@ static void doris_http_fetch_meta(struct doris_instance *instance)
static void instance_fetch_cfg_timer_cb(int fd, short kind, void *userp)
{
struct doris_instance *instance = (struct doris_instance *)userp;
struct doris_csum_instance *instance = (struct doris_csum_instance *)userp;
switch(instance->status)
{
@@ -482,7 +474,7 @@ static void instance_fetch_cfg_timer_cb(int fd, short kind, void *userp)
static void doris_client_fs_output_timer_cb(int fd, short kind, void *userp)
{
struct doris_parameter *param=(struct doris_parameter *)userp;
struct doris_csum_param *param=(struct doris_csum_param *)userp;
struct timeval tv;
FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_MST_CNN_SRV], 0, FS_OP_SET, param->param_master->connected_hosts);
@@ -503,7 +495,7 @@ static void doris_client_fs_output_timer_cb(int fd, short kind, void *userp)
evtimer_add(&param->fs_timer_output, &tv);
}
static int doris_client_register_field_stat(struct doris_parameter *param, void *runtime_log, struct event_base *evbase)
static int doris_client_register_field_stat(struct doris_csum_param *param, void *runtime_log, struct event_base *evbase)
{
const char *field_names[FSSTAT_DORIS_FILED_NUM]={"ReqFail", "ReqMetas", "ResMetas", "ResNoNew", "ReqFiles",
"ResFiles", "ResFrags", "ResFragErr", "ResBytes", "ResVerErr", "ReqBackup1", "ReqBackup2"};
@@ -549,19 +541,44 @@ static int doris_client_register_field_stat(struct doris_parameter *param, void
return 0;
}
struct doris_parameter *doris_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog)
u_int32_t doris_csum_param_get_refernces(struct doris_csum_param *param)
{
struct doris_parameter *param;
pthread_mutex_lock(&param->mutex_lock);
u_int32_t references = param->references;
pthread_mutex_unlock(&param->mutex_lock);
return references;
}
param = (struct doris_parameter *)calloc(1, sizeof(struct doris_parameter));
void doris_csum_parameter_destroy(struct doris_csum_param *param)
{
evtimer_del(&param->fs_timer_output);
FS_stop(&param->fsstat_handle);
doris_http_parameter_destroy(param->param_master);
if(param->param_backup1 != NULL)
{
doris_http_parameter_destroy(param->param_backup1);
}
if(param->param_backup2 != NULL)
{
doris_http_parameter_destroy(param->param_backup2);
}
free(param);
}
struct doris_csum_param *doris_csum_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog)
{
struct doris_csum_param *param;
param = (struct doris_csum_param *)calloc(1, sizeof(struct doris_csum_param));
param->manage_evbase = manage_evbase;
MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_fail_retry_interval", &param->retry_interval, 10);
MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_fragmet_size", &param->fetch_frag_size, 5242880);
MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_confile_max_tries", &param->fetch_max_tries, 3);
MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "master_slave_sync_on", &param->client_sync_on, 0);
MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_appname", param->fsstat_appname, 16, "DorisClient");
MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_filepath", param->fsstat_filepath, 256, "./log/doris_client.fs");
MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_appname", param->fsstat_appname, 16, "DrsCsmClient");
MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_filepath", param->fsstat_filepath, 256, "./log/doris_client_csm.fs");
MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fsstat_log_interval", &param->fsstat_period, 10);
MESA_load_profile_int_def(confile, "DORIS_CLIENT", "fsstat_log_print_mode", &param->fsstat_print_mode, 1);
MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_dst_ip", param->fsstat_dst_ip, 64, "127.0.0.1");
@@ -578,14 +595,15 @@ struct doris_parameter *doris_parameter_new(const char *confile, struct event_ba
}
param->param_backup1 = doris_http_parameter_new(confile, "DORIS_CLIENT.backup1_server", manage_evbase, runtimelog);
param->param_backup2 = doris_http_parameter_new(confile, "DORIS_CLIENT.backup2_server", manage_evbase, runtimelog);
pthread_mutex_init(&param->mutex_lock, NULL);
return param;
}
static void doris_instance_statistic_timer_cb(int fd, short kind, void *userp)
{
struct doris_instance *instance = (struct doris_instance *)userp;
struct doris_csum_instance *instance = (struct doris_csum_instance *)userp;
struct timeval tv;
struct doris_statistics incr_statistic;
struct doris_csum_statistics incr_statistic;
long long *plast_statistic = (long long*)&instance->statistic_last;
long long *pnow_statistic = (long long*)&instance->statistic;
long long *pinc_statistic = (long long*)&incr_statistic;
@@ -594,9 +612,9 @@ static void doris_instance_statistic_timer_cb(int fd, short kind, void *userp)
http_sessions += caculate_http_sessions_sum(instance->httpins_master);
http_sessions += caculate_http_sessions_sum(instance->httpins_backup1);
http_sessions += caculate_http_sessions_sum(instance->httpins_backup2);
instance->statistic.field[DRS_FS_STAT_HTTP_SESSIONS] = http_sessions;
instance->statistic.status[DRS_FS_STAT_HTTP_SESSIONS] = http_sessions;
for(u_int32_t i=0; i<sizeof(struct doris_statistics)/sizeof(long long); i++)
for(u_int32_t i=0; i<sizeof(struct doris_csum_statistics)/sizeof(long long); i++)
{
pinc_statistic[i] = pnow_statistic[i] - plast_statistic[i];
}
@@ -615,13 +633,38 @@ static void doris_instance_statistic_timer_cb(int fd, short kind, void *userp)
event_add(&instance->timer_statistic, &tv);
}
struct doris_instance *doris_instance_new(struct doris_parameter *param, struct event_base *worker_evbase,
struct doris_csum_param *doris_csum_instance_get_param(struct doris_csum_instance *instance)
{
return instance->param;
}
void doris_csum_instance_destroy(struct doris_csum_instance *instance)
{
pthread_mutex_lock(&instance->param->mutex_lock);
instance->param->references--;
pthread_mutex_unlock(&instance->param->mutex_lock);
evtimer_del(&instance->timer_fetch);
evtimer_del(&instance->timer_statistic);
/*doris_http_instance_destroy(instance->httpins_master);
if(instance->httpins_backup1 != NULL)
{
doris_http_instance_destroy(instance->httpins_backup1);
}
if(instance->httpins_backup2 != NULL)
{
doris_http_instance_destroy(instance->httpins_backup2);
}*/
free(instance);
}
struct doris_csum_instance *doris_csum_instance_new(struct doris_csum_param *param, struct event_base *worker_evbase,
struct doris_callbacks *cbs, struct doris_arguments *args, void *runtimelog)
{
struct doris_instance *instance;
struct doris_csum_instance *instance;
struct timeval tv;
instance = (struct doris_instance *)calloc(1, sizeof(struct doris_instance));
instance = (struct doris_csum_instance *)calloc(1, sizeof(struct doris_csum_instance));
instance->param = param;
instance->worker_evbase = worker_evbase;
instance->runtime_log = runtimelog;
@@ -646,6 +689,10 @@ struct doris_instance *doris_instance_new(struct doris_parameter *param, struct
instance->httpins_backup2 = doris_http_instance_new(param->param_backup2, worker_evbase, runtimelog);
}
pthread_mutex_lock(&param->mutex_lock);
param->references++;
pthread_mutex_unlock(&param->mutex_lock);
evtimer_assign(&instance->timer_statistic, worker_evbase, doris_instance_statistic_timer_cb, instance);
tv.tv_sec = param->fsstat_period;
tv.tv_usec = 0;