From 1aca701f127e94a7fbf60a996328083fed665f56 Mon Sep 17 00:00:00 2001 From: "linuxrc@163.com" Date: Wed, 25 Aug 2021 18:40:20 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0HTTP=20Post=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=8E=A5=E5=8F=A3=EF=BC=8C=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E4=B8=BB=E4=BB=8E=E5=8F=8C=E6=9C=BA=E5=A4=87=E4=BB=BD=E4=B8=8E?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=EF=BC=88=E5=86=B7=E5=A4=87=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/CMakeLists.txt | 2 +- client/doris_client_fetch.cpp | 169 +- client/doris_client_fetch.h | 21 +- client/doris_client_http.cpp | 83 + client/doris_client_http.h | 12 +- client/doris_client_produce.cpp | 518 +++++ client/doris_client_produce.h | 61 + client/doris_client_transfer.cpp | 183 +- client/doris_client_transfer.h | 4 + include/doris_client.h | 79 - include/doris_consumer_client.h | 86 + include/doris_producer_client.h | 80 + monitor/monitor_expamle.png | Bin 0 -> 35516 bytes .../配置分发网络运行状态-1626422695467.json | 106 +- server/CMakeLists.txt | 2 +- server/bin/conf/doris_client.conf | 35 - server/bin/conf/doris_client_csum.conf | 30 + server/bin/conf/doris_client_sync_t1.conf | 28 + server/bin/conf/doris_client_sync_voip.conf | 28 + server/bin/conf/doris_main.conf | 27 +- server/doris_server_http.cpp | 171 +- server/doris_server_http.h | 1 - server/doris_server_main.cpp | 214 +- server/doris_server_main.h | 58 +- server/doris_server_receive.cpp | 1841 +++++++++++++++-- server/doris_server_receive.h | 86 +- server/doris_server_scandir.cpp | 14 +- server/doris_server_scandir.h | 4 +- 28 files changed, 3295 insertions(+), 648 deletions(-) create mode 100644 client/doris_client_produce.cpp create mode 100644 client/doris_client_produce.h delete mode 100644 include/doris_client.h create mode 100644 include/doris_consumer_client.h create mode 100644 include/doris_producer_client.h create mode 100644 monitor/monitor_expamle.png delete mode 100644 server/bin/conf/doris_client.conf create mode 100644 server/bin/conf/doris_client_csum.conf create mode 100644 server/bin/conf/doris_client_sync_t1.conf create mode 100644 server/bin/conf/doris_client_sync_voip.conf diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index 9c13e61..1a8eb2a 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -1,4 +1,4 @@ -set (DORIS_CLIENT_SRC doris_client_fetch.cpp doris_client_http.cpp doris_client_transfer.cpp nirvana_conhash.cpp nirvana_murmurhash.cpp) +set (DORIS_CLIENT_SRC doris_client_fetch.cpp doris_client_produce.cpp doris_client_http.cpp doris_client_transfer.cpp nirvana_conhash.cpp nirvana_murmurhash.cpp) add_definitions(-fPIC -Wall -g) diff --git a/client/doris_client_fetch.cpp b/client/doris_client_fetch.cpp index 9e1d9ef..8827463 100644 --- a/client/doris_client_fetch.cpp +++ b/client/doris_client_fetch.cpp @@ -35,29 +35,6 @@ static int doris_md5_final_string(MD5_CTX *c, char *result, unsigned int size) return 0; } -void easy_string_destroy(struct easy_string *estr) -{ - if(estr->buff != NULL) - { - free(estr->buff); - estr->buff = NULL; - estr->len = estr->size = 0; - } -} - -void easy_string_savedata(struct easy_string *estr, const char *data, size_t len) -{ - if(estr->size-estr->len < len+1) - { - estr->size += len*4+1; - estr->buff = (char*)realloc(estr->buff, estr->size); - } - - memcpy(estr->buff+estr->len, data, len); - estr->len += len; - estr->buff[estr->len]='\0'; -} - void doris_confile_ctx_reset(struct doris_confile_ctx *ctx) { //ֲ @@ -75,12 +52,12 @@ void doris_confile_ctx_destry(struct doris_confile_ctx *ctx) ctx->httpctx = NULL; } -void doris_update_new_version(struct doris_instance *instance) +void doris_update_new_version(struct doris_csum_instance *instance) { instance->cur_version = instance->new_version; } -void doris_request_restart_timer(struct doris_instance *instance, time_t wait_s) +void doris_request_restart_timer(struct doris_csum_instance *instance, time_t wait_s) { struct timeval tv; @@ -89,7 +66,7 @@ void doris_request_restart_timer(struct doris_instance *instance, time_t wait_s) event_add(&instance->timer_fetch, &tv); } -void doris_fetch_next_confile_meta(struct doris_instance *instance) +void doris_fetch_next_confile_meta(struct doris_csum_instance *instance) { cJSON *cur_a_item, *sub; @@ -125,7 +102,7 @@ void doris_fetch_next_confile_meta(struct doris_instance *instance) void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code, long res_code, void *userp) { - struct doris_instance *instance = (struct doris_instance *)userp; + struct doris_csum_instance *instance = (struct doris_csum_instance *)userp; const char *pos_colon; size_t datalen; char buffer[64]; @@ -180,7 +157,7 @@ void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code void doris_http_confile_body_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp) { - struct doris_instance *instance = (struct doris_instance *)userp; + struct doris_csum_instance *instance = (struct doris_csum_instance *)userp; if(code!=CURLE_OK || (instance->ctx.res_code!=200 && instance->ctx.res_code!=206) || (res_code!=200 && res_code!=206)) { @@ -193,10 +170,10 @@ void doris_http_confile_body_cb(const char *ptr, size_t bytes, CURLcode code, lo instance->statistic.field[DRS_FS_FILED_RES_BYTES] += bytes; } -void doris_http_fetch_confile(struct doris_instance *instance); +void doris_http_fetch_confile(struct doris_csum_instance *instance); void doris_http_confile_done_cb(CURLcode res, long res_code, const char *err, void *userp) { - struct doris_instance *instance = (struct doris_instance *)userp; + struct doris_csum_instance *instance = (struct doris_csum_instance *)userp; char md5buffer[64]; bool direct_fail=false; @@ -284,7 +261,7 @@ out_md5: doris_request_restart_timer(instance, instance->param->retry_interval); } -void doris_http_fetch_confile(struct doris_instance *instance) +void doris_http_fetch_confile(struct doris_csum_instance *instance) { struct doris_http_callback curlcbs; char metauri[128], range[64]={0}; @@ -319,7 +296,7 @@ void doris_http_fetch_confile(struct doris_instance *instance) void doris_http_meta_header_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp) { - struct doris_instance *instance = (struct doris_instance *)userp; + struct doris_csum_instance *instance = (struct doris_csum_instance *)userp; //check code only once if(instance->ctx.res_code != 0) @@ -331,14 +308,14 @@ void doris_http_meta_header_cb(const char *ptr, size_t bytes, CURLcode code, lon if(res_code != 200) { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "No new meta found, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d", - instance->ctx.server, instance->cur_version, instance->req_version, code); + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "business: %s, No new meta found, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d", + instance->args.bizname, instance->ctx.server, instance->cur_version, instance->req_version, code); } } void doris_http_meta_body_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp) { - struct doris_instance *instance = (struct doris_instance *)userp; + struct doris_csum_instance *instance = (struct doris_csum_instance *)userp; if(code!=CURLE_OK || res_code!=200) { @@ -350,14 +327,14 @@ void doris_http_meta_body_cb(const char *ptr, size_t bytes, CURLcode code, long void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void *userp) { - struct doris_instance *instance = (struct doris_instance *)userp; + struct doris_csum_instance *instance = (struct doris_csum_instance *)userp; cJSON *sub; int64_t new_version; if(res!=CURLE_OK) { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Request meta failed, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s", - instance->ctx.server, instance->cur_version, instance->req_version, res_code, err); + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "business: %s, Request meta failed, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s", + instance->args.bizname, instance->ctx.server, instance->cur_version, instance->req_version, res_code, err); goto out_error; } @@ -369,16 +346,16 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void instance->meta = cJSON_Parse(instance->estr.buff); if(instance->meta == NULL) { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Parse meta failed, server: %s, req_version=%lu, invalid json: %s", - instance->ctx.server, instance->req_version, instance->estr.buff); + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "business: %s, Parse meta failed, server: %s, req_version=%lu, invalid json: %s", + instance->args.bizname, instance->ctx.server, instance->req_version, instance->estr.buff); goto out_error; } sub = cJSON_GetObjectItem(instance->meta, "version"); new_version = sub->valuedouble; if(new_version <= instance->cur_version) { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "An older version received, abandon it. server: %s, cur_version=%lu, invalid json: %s", - instance->ctx.server, instance->cur_version, instance->estr.buff); + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "business: %s, An older version received, abandon it. server: %s, cur_version=%lu, invalid json: %s", + instance->args.bizname, instance->ctx.server, instance->cur_version, instance->estr.buff); cJSON_Delete(instance->meta); instance->meta = NULL; goto out_error; @@ -386,8 +363,8 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void instance->new_version = new_version; instance->req_version = instance->new_version; instance->statistic.field[DRS_FS_FILED_RES_META] += 1; - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, server: %s, cur_version=%lu, newjson: %s", - instance->ctx.server, instance->cur_version, instance->estr.buff); + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "business: %s, NEW_META found, server: %s, cur_version=%lu, newjson: %s", + instance->args.bizname, instance->ctx.server, instance->cur_version, instance->estr.buff); instance->cbs.version_start(instance, instance->meta, instance->cbs.userdata); instance->array = cJSON_GetObjectItem(instance->meta, "configs"); @@ -405,13 +382,21 @@ out_error: doris_request_restart_timer(instance, instance->param->retry_interval); easy_string_destroy(&instance->estr); doris_confile_ctx_destry(&instance->ctx); + if(res_code==304 && instance->cbs.version_updated!=NULL) //汾ͬ + { + instance->cbs.version_updated(instance, instance->cbs.userdata); + } + if(res_code==300 && instance->param->client_sync_on) //а;еİ汾ϴ + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "business: %s, Server is busy processing version requests, waiting it done...", instance->args.bizname); + } } -static void doris_http_fetch_meta(struct doris_instance *instance) +static void doris_http_fetch_meta(struct doris_csum_instance *instance) { u_int64_t balance_seed; struct doris_http_callback curlcbs; - char metauri[128]; + char metauri[128], cur_version[128]; balance_seed = (((u_int64_t)rand()&0xFFFF) << 48) | (((u_int64_t)rand()&0xFFFF) << 32) | (((u_int64_t)rand()&0xFFFF) << 16) | ((u_int64_t)rand()&0xFFFF); @@ -439,6 +424,13 @@ static void doris_http_fetch_meta(struct doris_instance *instance) instance->req_version = instance->cur_version + 1; //ֻа汾³ɹcur_versionŻ if(instance->ctx.httpctx != NULL) { + if(instance->param->client_sync_on) + { + //ͬѣڱǴClient + sprintf(cur_version, "X-Doris-Sync-Current-Version: %lu", instance->cur_version); + doris_http_ctx_add_header(instance->ctx.httpctx, cur_version); + } + snprintf(metauri, 128, "configmeta?version=%lu&business=%s", instance->req_version, instance->args.bizname); if(!doris_http_launch_get_request(instance->ctx.httpctx, metauri)) { @@ -464,7 +456,7 @@ static void doris_http_fetch_meta(struct doris_instance *instance) static void instance_fetch_cfg_timer_cb(int fd, short kind, void *userp) { - struct doris_instance *instance = (struct doris_instance *)userp; + struct doris_csum_instance *instance = (struct doris_csum_instance *)userp; switch(instance->status) { @@ -482,7 +474,7 @@ static void instance_fetch_cfg_timer_cb(int fd, short kind, void *userp) static void doris_client_fs_output_timer_cb(int fd, short kind, void *userp) { - struct doris_parameter *param=(struct doris_parameter *)userp; + struct doris_csum_param *param=(struct doris_csum_param *)userp; struct timeval tv; FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_MST_CNN_SRV], 0, FS_OP_SET, param->param_master->connected_hosts); @@ -503,7 +495,7 @@ static void doris_client_fs_output_timer_cb(int fd, short kind, void *userp) evtimer_add(¶m->fs_timer_output, &tv); } -static int doris_client_register_field_stat(struct doris_parameter *param, void *runtime_log, struct event_base *evbase) +static int doris_client_register_field_stat(struct doris_csum_param *param, void *runtime_log, struct event_base *evbase) { const char *field_names[FSSTAT_DORIS_FILED_NUM]={"ReqFail", "ReqMetas", "ResMetas", "ResNoNew", "ReqFiles", "ResFiles", "ResFrags", "ResFragErr", "ResBytes", "ResVerErr", "ReqBackup1", "ReqBackup2"}; @@ -549,19 +541,44 @@ static int doris_client_register_field_stat(struct doris_parameter *param, void return 0; } -struct doris_parameter *doris_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog) +u_int32_t doris_csum_param_get_refernces(struct doris_csum_param *param) { - struct doris_parameter *param; + pthread_mutex_lock(¶m->mutex_lock); + u_int32_t references = param->references; + pthread_mutex_unlock(¶m->mutex_lock); + return references; +} - param = (struct doris_parameter *)calloc(1, sizeof(struct doris_parameter)); +void doris_csum_parameter_destroy(struct doris_csum_param *param) +{ + evtimer_del(¶m->fs_timer_output); + FS_stop(¶m->fsstat_handle); + doris_http_parameter_destroy(param->param_master); + if(param->param_backup1 != NULL) + { + doris_http_parameter_destroy(param->param_backup1); + } + if(param->param_backup2 != NULL) + { + doris_http_parameter_destroy(param->param_backup2); + } + free(param); +} + +struct doris_csum_param *doris_csum_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog) +{ + struct doris_csum_param *param; + + param = (struct doris_csum_param *)calloc(1, sizeof(struct doris_csum_param)); param->manage_evbase = manage_evbase; MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_fail_retry_interval", ¶m->retry_interval, 10); MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_fragmet_size", ¶m->fetch_frag_size, 5242880); MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_confile_max_tries", ¶m->fetch_max_tries, 3); + MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "master_slave_sync_on", ¶m->client_sync_on, 0); - MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_appname", param->fsstat_appname, 16, "DorisClient"); - MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_filepath", param->fsstat_filepath, 256, "./log/doris_client.fs"); + MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_appname", param->fsstat_appname, 16, "DrsCsmClient"); + MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_filepath", param->fsstat_filepath, 256, "./log/doris_client_csm.fs"); MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fsstat_log_interval", ¶m->fsstat_period, 10); MESA_load_profile_int_def(confile, "DORIS_CLIENT", "fsstat_log_print_mode", ¶m->fsstat_print_mode, 1); MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_dst_ip", param->fsstat_dst_ip, 64, "127.0.0.1"); @@ -578,14 +595,15 @@ struct doris_parameter *doris_parameter_new(const char *confile, struct event_ba } param->param_backup1 = doris_http_parameter_new(confile, "DORIS_CLIENT.backup1_server", manage_evbase, runtimelog); param->param_backup2 = doris_http_parameter_new(confile, "DORIS_CLIENT.backup2_server", manage_evbase, runtimelog); + pthread_mutex_init(¶m->mutex_lock, NULL); return param; } static void doris_instance_statistic_timer_cb(int fd, short kind, void *userp) { - struct doris_instance *instance = (struct doris_instance *)userp; + struct doris_csum_instance *instance = (struct doris_csum_instance *)userp; struct timeval tv; - struct doris_statistics incr_statistic; + struct doris_csum_statistics incr_statistic; long long *plast_statistic = (long long*)&instance->statistic_last; long long *pnow_statistic = (long long*)&instance->statistic; long long *pinc_statistic = (long long*)&incr_statistic; @@ -594,9 +612,9 @@ static void doris_instance_statistic_timer_cb(int fd, short kind, void *userp) http_sessions += caculate_http_sessions_sum(instance->httpins_master); http_sessions += caculate_http_sessions_sum(instance->httpins_backup1); http_sessions += caculate_http_sessions_sum(instance->httpins_backup2); - instance->statistic.field[DRS_FS_STAT_HTTP_SESSIONS] = http_sessions; + instance->statistic.status[DRS_FS_STAT_HTTP_SESSIONS] = http_sessions; - for(u_int32_t i=0; itimer_statistic, &tv); } -struct doris_instance *doris_instance_new(struct doris_parameter *param, struct event_base *worker_evbase, +struct doris_csum_param *doris_csum_instance_get_param(struct doris_csum_instance *instance) +{ + return instance->param; +} + +void doris_csum_instance_destroy(struct doris_csum_instance *instance) +{ + pthread_mutex_lock(&instance->param->mutex_lock); + instance->param->references--; + pthread_mutex_unlock(&instance->param->mutex_lock); + + evtimer_del(&instance->timer_fetch); + evtimer_del(&instance->timer_statistic); + /*doris_http_instance_destroy(instance->httpins_master); + if(instance->httpins_backup1 != NULL) + { + doris_http_instance_destroy(instance->httpins_backup1); + } + if(instance->httpins_backup2 != NULL) + { + doris_http_instance_destroy(instance->httpins_backup2); + }*/ + free(instance); +} + +struct doris_csum_instance *doris_csum_instance_new(struct doris_csum_param *param, struct event_base *worker_evbase, struct doris_callbacks *cbs, struct doris_arguments *args, void *runtimelog) { - struct doris_instance *instance; + struct doris_csum_instance *instance; struct timeval tv; - instance = (struct doris_instance *)calloc(1, sizeof(struct doris_instance)); + instance = (struct doris_csum_instance *)calloc(1, sizeof(struct doris_csum_instance)); instance->param = param; instance->worker_evbase = worker_evbase; instance->runtime_log = runtimelog; @@ -646,6 +689,10 @@ struct doris_instance *doris_instance_new(struct doris_parameter *param, struct instance->httpins_backup2 = doris_http_instance_new(param->param_backup2, worker_evbase, runtimelog); } + pthread_mutex_lock(¶m->mutex_lock); + param->references++; + pthread_mutex_unlock(¶m->mutex_lock); + evtimer_assign(&instance->timer_statistic, worker_evbase, doris_instance_statistic_timer_cb, instance); tv.tv_sec = param->fsstat_period; tv.tv_usec = 0; diff --git a/client/doris_client_fetch.h b/client/doris_client_fetch.h index 58376ba..a236135 100644 --- a/client/doris_client_fetch.h +++ b/client/doris_client_fetch.h @@ -1,19 +1,13 @@ #ifndef __DORIS_CLIENT_FETCH_IN_H__ #define __DORIS_CLIENT_FETCH_IN_H__ +#include #include #include -#include "doris_client.h" +#include "doris_consumer_client.h" #include "doris_client_http.h" -struct easy_string -{ - char* buff; - size_t len; - size_t size; -}; - enum FETCH_CFG_STATUS { FETCH_STATUS_IDLE=0, @@ -21,11 +15,14 @@ enum FETCH_CFG_STATUS FETCH_STATUS_FILE, }; -struct doris_parameter +struct doris_csum_param { u_int32_t retry_interval; u_int32_t fetch_frag_size; u_int32_t fetch_max_tries; + u_int32_t client_sync_on; + pthread_mutex_t mutex_lock; + u_int32_t references; struct doris_http_parameter *param_master; struct doris_http_parameter *param_backup1; @@ -76,7 +73,7 @@ struct doris_confile_ctx size_t contl_total; }; -struct doris_instance +struct doris_csum_instance { struct doris_callbacks cbs; struct doris_arguments args; @@ -102,9 +99,9 @@ struct doris_instance struct event_base *worker_evbase; struct event timer_fetch; - struct doris_parameter *param; + struct doris_csum_param *param; struct event timer_statistic; - struct doris_statistics statistic, statistic_last; + struct doris_csum_statistics statistic, statistic_last; void *runtime_log; }; diff --git a/client/doris_client_http.cpp b/client/doris_client_http.cpp index fbd9181..a63ac91 100644 --- a/client/doris_client_http.cpp +++ b/client/doris_client_http.cpp @@ -21,6 +21,38 @@ #include "doris_client_http.h" +void easy_string_destroy(struct easy_string *estr) +{ + if(estr->buff != NULL) + { + free(estr->buff); + estr->buff = NULL; + estr->len = estr->size = 0; + } +} + +void easy_string_savedata(struct easy_string *estr, const char *data, size_t len) +{ + if(estr->size-estr->len < len+1) + { + estr->size += len*4+1; + estr->buff = (char*)realloc(estr->buff, estr->size); + } + + memcpy(estr->buff+estr->len, data, len); + estr->len += len; + estr->buff[estr->len]='\0'; +} + +static inline void drsclient_set_sockopt_keepalive(int sd, int keepidle, int keepintvl, int keepcnt) +{ + int keepalive = 1; + setsockopt(sd, SOL_SOCKET, SO_KEEPALIVE, (void*)&keepalive, sizeof(keepalive)); + setsockopt(sd, SOL_TCP, TCP_KEEPIDLE, (void*)&keepidle, sizeof(keepidle)); + setsockopt(sd, SOL_TCP, TCP_KEEPINTVL, (void*)&keepintvl, sizeof(keepintvl)); + setsockopt(sd, SOL_TCP, TCP_KEEPCNT, (void*)&keepcnt, sizeof(keepcnt)); +} + int32_t param_get_connected_hosts(struct doris_http_parameter *param) { return param->connected_hosts; @@ -199,6 +231,7 @@ static void client_bufferevent_error_cb(struct bufferevent *bev, short event, vo conhash_insert_dest_host(balance); assert(balance->param->connected_hosts > 0); assert(balance->param->failed_hosts >= 0); + drsclient_set_sockopt_keepalive(bufferevent_getfd(bev), 10, 5, 2); } else { @@ -303,6 +336,24 @@ static int32_t doris_launch_group_connection(struct doris_http_parameter *param, return 0; } +void doris_http_parameter_destroy(struct doris_http_parameter *param) +{ + for(u_int32_t i=0; iipgroup.dstaddr_num; i++) // + { + if(evtimer_pending(¶m->balance[i].timer_detect, NULL)) + { + evtimer_del(¶m->balance[i].timer_detect); + } + if(param->balance[i].bev != NULL) + { + bufferevent_free(param->balance[i].bev); + } + } + conhash_instance_free(param->conhash); + free(param->ipgroup.dstaddrs); + free(param); +} + struct doris_http_parameter *doris_http_parameter_new(const char* profile_path, const char* section, struct event_base* evbase, void *runtime_log) { struct doris_http_parameter *param; @@ -352,6 +403,38 @@ struct doris_http_parameter *doris_http_parameter_new(const char* profile_path, return param; } +void doris_http_instance_destroy(struct doris_http_instance *instance) +{ + map::iterator iter; + struct doris_curl_multihd *multihd; + CURLMsg *msg; + int msgs_left; + struct doris_http_ctx *ctx; + CURL *easy; + + for(iter=instance->server_hosts->begin(); iter!=instance->server_hosts->end(); ) + { + multihd = iter->second; + + while((msg = curl_multi_info_read(multihd->multi_hd, &msgs_left))) + { + easy = msg->easy_handle; + curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ctx); + curl_multi_remove_handle(multihd->multi_hd, easy); + curl_easy_cleanup(easy); + ctx->curl = NULL; + ctx->transfering = 0; + ctx->res = CURLE_ABORTED_BY_CALLBACK; + ctx->res_code = 0; + ctx->cb.transfer_done_cb(ctx->res, 0, ctx->error, ctx->cb.userp); + } + curl_multi_cleanup(multihd->multi_hd); + + instance->server_hosts->erase(iter++); + } + free(instance); +} + struct doris_http_instance *doris_http_instance_new(struct doris_http_parameter *param, struct event_base* evbase, void *runtimelog) { struct doris_http_instance *instance; diff --git a/client/doris_client_http.h b/client/doris_client_http.h index 6f829ab..4dd3550 100644 --- a/client/doris_client_http.h +++ b/client/doris_client_http.h @@ -28,6 +28,15 @@ using namespace std; #define DEFAULT_HOST_CAPACITY 4 #define LOAD_BALANC_VIRT_TIMES 16 +struct easy_string +{ + char* buff; + size_t len; + size_t size; +}; +void easy_string_destroy(struct easy_string *estr); +void easy_string_savedata(struct easy_string *estr, const char *data, size_t len); + enum TCP_CONNECTION_STATUS { TCP_STATUS_IDLE=0, @@ -83,7 +92,6 @@ struct doris_http_parameter struct doris_http_instance { struct event_base* evbase; - SSL_CTX *ssl_instance; void *privdata; map *server_hosts; @@ -102,7 +110,9 @@ int32_t param_get_connected_hosts(struct doris_http_parameter *param); int32_t param_get_failed_hosts(struct doris_http_parameter *param); struct doris_http_parameter *doris_http_parameter_new(const char* profile_path, const char* section, struct event_base* evbase, void *runtime_log); +void doris_http_parameter_destroy(struct doris_http_parameter *param); struct doris_http_instance *doris_http_instance_new(struct doris_http_parameter *param, struct event_base* evbase, void *runtimelog); +void doris_http_instance_destroy(struct doris_http_instance *instance); #endif diff --git a/client/doris_client_produce.cpp b/client/doris_client_produce.cpp new file mode 100644 index 0000000..d8fcedc --- /dev/null +++ b/client/doris_client_produce.cpp @@ -0,0 +1,518 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "doris_client_produce.h" + +void doris_prod_upload_ctx_destroy(struct doris_upload_ctx *ctx) +{ + doris_http_ctx_destroy(ctx->httpctx); + free(ctx); +} + +static enum PROD_VEROP_RES version_common_result_assign_val(CURLcode res, long res_code) +{ + if(res != CURLE_OK) + { + return VERSIONOP_CURL_ERROR; + } + + switch(res_code) + { + case 200: return VERSIONOP_RES_OK; + case 201: return VERSIONOP_RES_OK; //ļֶظϴŻ᷵201 + default: return VERSIONOP_RES_ERROR; + } +} + +void version_cancel_transfer_done_cb(CURLcode res, long res_code, const char *err, void *userp) +{ + struct doris_upload_ctx *ctx=(struct doris_upload_ctx *)userp; + enum PROD_VEROP_RES result; + + ctx->instance->statistic.status[DRS_FSPRD_STAT_REQ_SESSIONS] -= 1; + + result = version_common_result_assign_val(res, res_code); + + if(result != VERSIONOP_RES_OK) + { + MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_FATAL, "business: %s, version cancel sync failed, res_code: %ld, err: %s", ctx->business, res_code, err); + } + else + { + MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "business: %s, version cancel sync succ, res_code: %ld", ctx->business, res_code); + } + + if(ctx->vercancel_cb != NULL) + { + ctx->vercancel_cb(result, ctx->userdata); + } +} + +int32_t doris_prod_version_cancel(struct doris_upload_ctx *ctx, void (*vercancel_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata) +{ + struct doris_http_callback cb; + char uri[256]; + + ctx->instance->statistic.field[DRS_FSPRD_FILED_VERCANCEL] += 1; + ctx->instance->statistic.status[DRS_FSPRD_STAT_REQ_SESSIONS] += 1; + + ctx->vercancel_cb = vercancel_cb; + ctx->userdata = userdata; + + cb.userp = ctx; + cb.header_cb = NULL; + cb.write_cb = NULL; + cb.read_process_cb = NULL; + cb.transfer_done_cb = version_cancel_transfer_done_cb; + doris_http_ctx_reset(ctx->httpctx, &cb); + + if(ctx->instance->param->client_sync_on) + { + doris_http_ctx_add_header(ctx->httpctx, "X-Doris-Master-Slave-Sync: 1"); + } + snprintf(uri, sizeof(uri), "version/cancel?token=%s", ctx->token); + return doris_http_launch_post_request(ctx->httpctx, uri, NULL, 0); +} + +void version_end_header_cb(const char *start, size_t bytes, CURLcode code, long res_code, void *userp) +{ + struct doris_upload_ctx *ctx=(struct doris_upload_ctx *)userp; + const char *pos_colon; + char buffer[64]; + int datalen; + + if((pos_colon=(const char*)memchr(start, ':', bytes)) == NULL) + { + return ; + } + datalen = pos_colon - start; + switch(datalen) + { + case 13: + if(!strncasecmp(start, "X-Set-Version:", 14)) + { + memcpy(buffer, start+14, bytes-14); + buffer[bytes-14] = '\0'; + ctx->res_version = atol(buffer); + } + break; + default: break; + } +} + +void version_end_transfer_done_cb(CURLcode res, long res_code, const char *err, void *userp) +{ + struct doris_upload_ctx *ctx=(struct doris_upload_ctx *)userp; + enum PROD_VEROP_RES result; + + ctx->instance->statistic.status[DRS_FSPRD_STAT_REQ_SESSIONS] -= 1; + + result = version_common_result_assign_val(res, res_code); + + if(result != VERSIONOP_RES_OK) + { + MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_FATAL, "business: %s, version end sync failed, res_code: %ld, err: %s", ctx->business, res_code, err); + } + else + { + assert(ctx->res_version != 0); + MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_INFO, "business: %s, version end sync succ, res_code: %ld, new version: %lu", ctx->business, res_code, ctx->res_version); + } + + if(ctx->verend_cb != NULL) + { + ctx->verend_cb(result, ctx->res_version, ctx->userdata); + } +} + +int32_t doris_prod_version_end(struct doris_upload_ctx *ctx, + void (*verend_cb)(enum PROD_VEROP_RES result, int64_t version, void *userdata), void *userdata) +{ + struct doris_http_callback cb; + char uri[256]; + + ctx->instance->statistic.field[DRS_FSPRD_FILED_VEREND] += 1; + ctx->instance->statistic.status[DRS_FSPRD_STAT_REQ_SESSIONS] += 1; + + ctx->verend_cb = verend_cb; + ctx->userdata = userdata; + + cb.userp = ctx; + cb.header_cb = version_end_header_cb; + cb.write_cb = NULL; + cb.read_process_cb = NULL; + cb.transfer_done_cb = version_end_transfer_done_cb; + doris_http_ctx_reset(ctx->httpctx, &cb); + + if(ctx->instance->param->client_sync_on) + { + doris_http_ctx_add_header(ctx->httpctx, "X-Doris-Master-Slave-Sync: 1"); + } + snprintf(uri, sizeof(uri), "version/finish?token=%s", ctx->token); + return doris_http_launch_post_request(ctx->httpctx, uri, NULL, 0); +} + +void upload_frag_transfer_done_cb(CURLcode res, long res_code, const char *err, void *userp) +{ + struct doris_upload_ctx *ctx=(struct doris_upload_ctx *)userp; + enum PROD_VEROP_RES result; + + ctx->instance->statistic.status[DRS_FSPRD_STAT_REQ_SESSIONS] -= 1; + + result = version_common_result_assign_val(res, res_code); + + if(result != VERSIONOP_RES_OK) + { + MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_FATAL, "business: %s, upload frag sync failed, res_code: %ld, err: %s", ctx->business, res_code, err); + } + else + { + MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "business: %s, upload frag sync succ, filename: %s, offset: %lu", ctx->business, ctx->filename, ctx->req_offset); + } + + if(ctx->upfrag_cb != NULL) + { + ctx->upfrag_cb(result, ctx->userdata); + } +} + +int32_t do_doris_prod_upload_with_cb(struct doris_upload_ctx *ctx, char *data, size_t size, + struct table_meta *meta, const char *uri) +{ + struct doris_http_callback cb; + + ctx->instance->statistic.status[DRS_FSPRD_STAT_REQ_SESSIONS] += 1; + + cb.userp = ctx; + cb.header_cb = NULL; + cb.write_cb = NULL; + cb.read_process_cb = NULL; + cb.transfer_done_cb = upload_frag_transfer_done_cb; + doris_http_ctx_reset(ctx->httpctx, &cb); + + if(ctx->instance->param->client_sync_on) + { + doris_http_ctx_add_header(ctx->httpctx, "X-Doris-Master-Slave-Sync: 1"); + } + if(meta->userregion != NULL) + { + doris_http_ctx_add_header_kvstr(ctx->httpctx, "X-User-Info", meta->userregion); + } + doris_http_ctx_add_header_kvint(ctx->httpctx, "X-Config-Num", meta->cfgnum); + doris_http_ctx_add_header_kvstr(ctx->httpctx, "Content-MD5", meta->md5); + + return doris_http_launch_put_request_data(ctx->httpctx, uri, data, size); +} + +int32_t doris_prod_upload_frag_with_cb(struct doris_upload_ctx *ctx, char *data, size_t size, size_t offset, + bool last, struct table_meta *meta, void (*upfrag_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata) +{ + char uri[256]; + + ctx->instance->statistic.field[DRS_FSPRD_FILED_FILEFRAG] += 1; + + ctx->upfrag_cb = upfrag_cb; + ctx->userdata = userdata; + ctx->req_offset = offset; + snprintf(ctx->filename, 256, "%s", ctx->filename); + + if(last) + { + snprintf(uri, sizeof(uri), "filefrag/upload?token=%s&tablename=%s&filename=%s&offset=%lu&last=true", + ctx->token, meta->tablename, meta->filename, offset); + } + else + { + snprintf(uri, sizeof(uri), "filefrag/upload?token=%s&tablename=%s&filename=%s&offset=%lu", + ctx->token, meta->tablename, meta->filename, offset); + } + return do_doris_prod_upload_with_cb(ctx, data, size, meta, uri); +} + +int32_t doris_prod_upload_once_with_cb(struct doris_upload_ctx *ctx, char *data, size_t size, + struct table_meta *meta, void (*upfrag_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata) +{ + char uri[256]; + + ctx->instance->statistic.field[DRS_FSPRD_FILED_FILEONCE] += 1; + + ctx->upfrag_cb = upfrag_cb; + ctx->userdata = userdata; + ctx->req_offset = 0; + snprintf(ctx->filename, 256, "%s", ctx->filename); + + snprintf(uri, sizeof(uri), "fileonce/upload?token=%s&tablename=%s&filename=%s", + ctx->token, meta->tablename, meta->filename); + return do_doris_prod_upload_with_cb(ctx, data, size, meta, uri); +} + +void verstart_body_write_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp) +{ + struct doris_upload_ctx *ctx=(struct doris_upload_ctx *)userp; + + easy_string_savedata(&ctx->estr, (const char*)ptr, bytes); +} + +static enum PROD_VERSTART_RES version_start_result_assign_val(CURLcode res, long res_code) +{ + if(res != CURLE_OK) + { + return VERSTART_CURL_ERROR; + } + + switch(res_code) + { + case 200: return VERSTART_RES_OK; + case 300: return VERSTART_RES_BUSY; + default: return VERSTART_RES_ERROR; + } +} + +void verstart_transfer_done_cb(CURLcode res, long res_code, const char *err, void *userp) +{ + struct doris_upload_ctx *ctx=(struct doris_upload_ctx *)userp; + cJSON *meta, *token; + enum PROD_VERSTART_RES result; + + ctx->instance->statistic.status[DRS_FSPRD_STAT_REQ_SESSIONS] -= 1; + + result = version_start_result_assign_val(res, res_code); + switch(result) + { + case VERSTART_RES_OK: + case VERSTART_RES_BUSY: //server300tokenֻԼһЧ(Լ֮ǰ˻ָ) + meta = cJSON_Parse(ctx->estr.buff); + token = cJSON_GetObjectItem(meta, "token"); + assert(token->valuestring != NULL); + snprintf(ctx->token, 64, "%s", token->valuestring); + cJSON_Delete(meta); + MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_DEBUG, "business: %s, version start sync %s, res_code: %ld, body: %s", + (result==VERSTART_RES_OK)?"succ":"busy", ctx->business, res_code, ctx->estr.buff); + break; + + case VERSTART_RES_ERROR: + case VERSTART_CURL_ERROR: + MESA_HANDLE_RUNTIME_LOGV2(ctx->instance->runtime_log, RLOG_LV_FATAL, "business: %s, version start sync failed, res_code: %ld, err: %s", + ctx->business, res_code, err); + break; + default: assert(0);break; + } + + if(ctx->verstart_cb != NULL) + { + ctx->verstart_cb(result, ctx->estr.buff, ctx->userdata); + } + easy_string_destroy(&ctx->estr); +} + +struct doris_upload_ctx *doris_prod_upload_ctx_new(struct doris_prod_instance *instance,const char *business, int32_t cfgtype) +{ + struct doris_upload_ctx *ctx; + struct doris_http_callback cb; + + if(cfgtype!=1 && cfgtype!=2) + { + return NULL; + } + ctx = (struct doris_upload_ctx *)calloc(1, sizeof(struct doris_upload_ctx)); + snprintf(ctx->business, 32, "%s", business); + ctx->instance = instance; + ctx->cfg_type = cfgtype; + + cb.userp = ctx; + cb.header_cb = NULL; + cb.write_cb = verstart_body_write_cb; + cb.read_process_cb = NULL; + cb.transfer_done_cb = verstart_transfer_done_cb; + if(NULL == (ctx->httpctx = doris_http_ctx_new(instance->http_instance, &cb, rand(), NULL, 0))) + { + free(ctx); + return NULL; + } + return ctx; +} + +int32_t doris_prod_version_start_with_cb(struct doris_upload_ctx *ctx, + void (*verstart_cb)(enum PROD_VERSTART_RES result, const char *body, void *userdata), void *userdata) +{ + char uri[256]; + + ctx->userdata = userdata; + ctx->verstart_cb = verstart_cb; + + if(ctx->instance->param->client_sync_on) + { + doris_http_ctx_add_header(ctx->httpctx, "X-Doris-Master-Slave-Sync: 1"); + } + + snprintf(uri, sizeof(uri), "version/start?business=%s&type=%d", ctx->business, ctx->cfg_type); + if(doris_http_launch_post_request(ctx->httpctx, uri, NULL, 0)) + { + free(ctx); + return -1; + } + ctx->instance->statistic.field[DRS_FSPRD_FILED_VERSTART] += 1; + ctx->instance->statistic.status[DRS_FSPRD_STAT_REQ_SESSIONS] += 1; + return 0; +} + +int32_t doris_prod_version_start(struct doris_upload_ctx *ctx) +{ + return doris_prod_version_start_with_cb(ctx, NULL, NULL); +} + +static void doris_prod_fsoutput_timer_cb(int fd, short kind, void *userp) +{ + struct doris_prod_param *param=(struct doris_prod_param *)userp; + struct timeval tv; + + FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FSPRD_STAT_CNNED_SERVERS], 0, FS_OP_SET, param->param->connected_hosts); + FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FSPRD_STAT_FAILED_SERVERS], 0, FS_OP_SET, param->param->failed_hosts); + FS_passive_output(param->fsstat_handle); + + tv.tv_sec = param->fsstat_period; + tv.tv_usec = 0; + evtimer_add(¶m->fs_timer_output, &tv); +} + +static int doris_prod_register_fsstat(struct doris_prod_param *param, void *runtime_log, struct event_base *evbase) +{ + const char *field_names[FSSTAT_DORIS_PRD_FILED_NUM]={"VerStart", "VerEnd", "VerCancel", "FileFrag", "FileOnce"}; + const char *status_names[FSSTAT_DORIS_PRD_STATUS_NUM]={"ServerCnned", "ServerFail", "MemoryUsed", "HttpSession", "ReqSession"}; + struct timeval tv; + int value; + + param->fsstat_handle = FS_create_handle(); + FS_set_para(param->fsstat_handle, OUTPUT_DEVICE, param->fsstat_filepath, strlen(param->fsstat_filepath)+1); + if(param->fsstat_print_mode == 1) + { + FS_set_para(param->fsstat_handle, PRINT_MODE, ¶m->fsstat_print_mode, sizeof(param->fsstat_print_mode)); + } + else + { + FS_set_para(param->fsstat_handle, PRINT_MODE, ¶m->fsstat_print_mode, sizeof(param->fsstat_print_mode)); + value = 1; + FS_set_para(param->fsstat_handle, FLUSH_BY_DATE, &value, sizeof(value)); + } + value = param->fsstat_period; + FS_set_para(param->fsstat_handle, STAT_CYCLE, &value, sizeof(value)); + value = 0; + FS_set_para(param->fsstat_handle, CREATE_THREAD, &value, sizeof(value)); + FS_set_para(param->fsstat_handle, APP_NAME, param->fsstat_appname, strlen(param->fsstat_appname)+1); + FS_set_para(param->fsstat_handle, STATS_SERVER_IP, param->fsstat_dst_ip, strlen(param->fsstat_dst_ip)+1); + FS_set_para(param->fsstat_handle, STATS_SERVER_PORT, ¶m->fsstat_dst_port, sizeof(param->fsstat_dst_port)); + + for(int i=0; ifsstat_field[i] = FS_register(param->fsstat_handle, FS_STYLE_FIELD, FS_CALC_CURRENT, field_names[i]); + } + for(int i=0; ifsstat_status[i] = FS_register(param->fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, status_names[i]); + } + FS_start(param->fsstat_handle); + + evtimer_assign(¶m->fs_timer_output, evbase, doris_prod_fsoutput_timer_cb, param); + tv.tv_sec = param->fsstat_period; + tv.tv_usec = 0; + evtimer_add(¶m->fs_timer_output, &tv); + return 0; +} + +struct doris_prod_param *doris_prod_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog) +{ + struct doris_prod_param *param; + + param = (struct doris_prod_param *)calloc(1, sizeof(struct doris_prod_param)); + param->manage_evbase = manage_evbase; + + MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "upload_fragmet_size", ¶m->upload_frag_size, 5242880); + MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "master_slave_sync_on", ¶m->client_sync_on, 0); + + MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_appname", param->fsstat_appname, 16, "DrsPrdClient"); + MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_filepath_p", param->fsstat_filepath, 256, "./log/doris_client_prd.fs"); + MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fsstat_log_interval", ¶m->fsstat_period, 10); + MESA_load_profile_int_def(confile, "DORIS_CLIENT", "fsstat_log_print_mode", ¶m->fsstat_print_mode, 1); + MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_dst_ip", param->fsstat_dst_ip, 64, "127.0.0.1"); + MESA_load_profile_int_def(confile, "DORIS_CLIENT", "fsstat_log_dst_port", ¶m->fsstat_dst_port, 8125); + + /*ͬʱֻ˫*/ + param->param = doris_http_parameter_new(confile, "DORIS_CLIENT.produce", manage_evbase, runtimelog); + if(param->param == NULL) + { + return NULL; + } + assert(param->param->ipgroup.dstaddr_num == 1); + if(doris_prod_register_fsstat(param, runtimelog, manage_evbase)) + { + return NULL; + } + return param; +} + +static void doris_prod_statistic_timer_cb(int fd, short kind, void *userp) +{ + struct doris_prod_instance *instance = (struct doris_prod_instance *)userp; + struct timeval tv; + struct doris_prod_statistics incr_statistic; + long long *plast_statistic = (long long*)&instance->statistic_last; + long long *pnow_statistic = (long long*)&instance->statistic; + long long *pinc_statistic = (long long*)&incr_statistic; + + instance->statistic.status[DRS_FSPRD_STAT_HTTP_SESSIONS] = caculate_http_sessions_sum(instance->http_instance); + + for(u_int32_t i=0; istatistic_last = instance->statistic; + + for(u_int32_t i=0; iparam->fsstat_handle, instance->param->fsstat_field[i], 0, FS_OP_ADD, incr_statistic.field[i]); + } + for(u_int32_t i=0; iparam->fsstat_handle, instance->param->fsstat_status[i], 0, FS_OP_ADD, incr_statistic.status[i]); + } + tv.tv_sec = instance->param->fsstat_period; + tv.tv_usec = 0; + event_add(&instance->timer_statistic, &tv); +} + +struct doris_prod_instance *doris_prod_instance_new(struct doris_prod_param *param, struct event_base *worker_evbase, void *runtimelog) +{ + struct doris_prod_instance *instance; + struct timeval tv; + + instance = (struct doris_prod_instance *)calloc(1, sizeof(struct doris_prod_instance)); + instance->param = param; + instance->worker_evbase = worker_evbase; + instance->runtime_log = runtimelog; + + instance->http_instance = doris_http_instance_new(param->param, worker_evbase, runtimelog); + if(instance->http_instance == NULL) + { + return NULL; + } + srand((int64_t)param); + + evtimer_assign(&instance->timer_statistic, worker_evbase, doris_prod_statistic_timer_cb, instance); + tv.tv_sec = param->fsstat_period; + tv.tv_usec = 0; + evtimer_add(&instance->timer_statistic, &tv); + return instance; +} + diff --git a/client/doris_client_produce.h b/client/doris_client_produce.h new file mode 100644 index 0000000..26cf87e --- /dev/null +++ b/client/doris_client_produce.h @@ -0,0 +1,61 @@ +#ifndef __DORIS_CLIENT_PRODUCE_H__ +#define __DORIS_CLIENT_PRODUCE_H__ + +#include +#include + +#include "doris_producer_client.h" +#include "doris_client_http.h" + +struct doris_prod_param +{ + u_int32_t upload_frag_size; + u_int32_t client_sync_on; + + struct doris_http_parameter *param; + struct event_base *manage_evbase; + + screen_stat_handle_t fsstat_handle; + struct event fs_timer_output; + char fsstat_dst_ip[64]; + char fsstat_appname[16]; + char fsstat_filepath[256]; + u_int32_t fsstat_period; + int32_t fsstat_print_mode; + int32_t fsstat_dst_port; + int32_t fsstat_field[FSSTAT_DORIS_PRD_FILED_NUM]; + int32_t fsstat_status[FSSTAT_DORIS_PRD_STATUS_NUM]; +}; + +struct doris_prod_instance +{ + struct doris_prod_param *param; + struct doris_http_instance *http_instance; + + struct event_base *worker_evbase; + struct event timer_statistic; + struct doris_prod_statistics statistic, statistic_last; + void *runtime_log; +}; + +struct doris_upload_ctx +{ + struct doris_http_ctx *httpctx; + char token[64]; + char business[32]; + char filename[256]; + struct easy_string estr; + int32_t cfg_type; + size_t req_offset; + int64_t res_version; + + void *userdata; + void (*verstart_cb)(enum PROD_VERSTART_RES result, const char *body, void *userdata); + void (*upfrag_cb)(enum PROD_VEROP_RES result, void *userdata); + void (*verend_cb)(enum PROD_VEROP_RES result, int64_t version, void *userdata); + void (*vercancel_cb)(enum PROD_VEROP_RES result, void *userdata); + struct doris_prod_instance *instance; +}; + +#endif + diff --git a/client/doris_client_transfer.cpp b/client/doris_client_transfer.cpp index d26bba2..b0edcf2 100644 --- a/client/doris_client_transfer.cpp +++ b/client/doris_client_transfer.cpp @@ -50,7 +50,7 @@ void doris_http_ctx_destroy(struct doris_http_ctx *ctx) } struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, - struct doris_http_callback *cb, u_int64_t balance_seed, char *host, int32_t size) + struct doris_http_callback *cb, u_int64_t balance_seed, char *host/*OUT*/, int32_t size) { struct doris_http_ctx *ctx; struct doris_curl_multihd *multidata; @@ -62,8 +62,10 @@ struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, } assert(instance->server_hosts->find(result.bucket_id) != instance->server_hosts->end()); multidata = instance->server_hosts->find(result.bucket_id)->second; - snprintf(host, size, multidata->host->srvaddr); - + if(host != NULL) + { + snprintf(host, size, multidata->host->srvaddr); + } ctx = (struct doris_http_ctx *)calloc(1, sizeof(struct doris_http_ctx)); ctx->instance = instance; ctx->multidata = multidata; @@ -92,8 +94,8 @@ static inline void curl_set_common_options(CURL *curl, long transfer_timeout, ch curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 1000L); curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //Էֶijӽտס - //ctx->error="Operation too slow. Less than 100 bytes/sec transferred the last 10 seconds" - curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 10L); + //ctx->error="Operation too slow. Less than 100 bytes/sec transferred the last 30 seconds" + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L); curl_easy_setopt(curl, CURLOPT_USERAGENT, "Doris Client Linux X64"); } @@ -136,6 +138,21 @@ void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header) ctx->headers = curl_slist_append(ctx->headers, header); } +/*maximum length 1024*/ +void doris_http_ctx_add_header_kvstr(struct doris_http_ctx *ctx, const char *headername, const char *value) +{ + char header[1024]; + snprintf(header, 1024, "%s: %s", headername, value); + ctx->headers = curl_slist_append(ctx->headers, header); +} + +void doris_http_ctx_add_header_kvint(struct doris_http_ctx *ctx, const char *headername, u_int64_t value) +{ + char header[1024]; + snprintf(header, 1024, "%s: %lu", headername, value); + ctx->headers = curl_slist_append(ctx->headers, header); +} + int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri) { char minio_url[2048]; @@ -143,7 +160,7 @@ int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri) assert(ctx->curl == NULL); if(NULL == (ctx->curl=curl_easy_init())) { - return -1; + assert(0);return -1; } if(ctx->instance->param->ssl_connection) @@ -171,8 +188,7 @@ int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri) if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl)) { - assert(0); - return -2; + assert(0);return -2; } ctx->transfering = 1; return 0; @@ -185,7 +201,7 @@ int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, assert(ctx->curl == NULL); if(NULL == (ctx->curl=curl_easy_init())) { - return -1; + assert(0);return -1; } doris_http_ctx_add_header(ctx, "Expect:"); //עPOSTExpectϵҪȷCURLOPT_POSTFIELDSIZE @@ -219,8 +235,153 @@ int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl)) { - assert(0); - return -2; + assert(0);return -2; + } + ctx->transfering = 1; + return 0; +} + +static size_t curl_put_data_request_send_cb(void *ptr, size_t size, size_t count, void *userp) +{ + size_t len; + struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp; + + if(size==0 || count==0 || ctx->put_offset>=ctx->put_length) + { + return 0; //һ + } + + len = ctx->put_length - ctx->put_offset; //ʣϴij + if(len > size * count) + { + len = size * count; + } + + memcpy(ptr, ctx->put_data + ctx->put_offset, len); + ctx->put_offset += len; + + if(ctx->cb.read_process_cb != NULL) + { + ctx->cb.read_process_cb(ctx->put_data, ctx->put_offset, ctx->cb.userp); + } + return len; +} + +int doris_http_launch_put_request_data(struct doris_http_ctx *ctx, const char *uri, char *data, size_t data_len) +{ + char minio_url[2048]; + + assert(ctx->curl == NULL); + if(NULL == (ctx->curl=curl_easy_init())) + { + assert(0);return -1; + } + ctx->put_data = data; + ctx->put_length = data_len; + ctx->put_offset = 0; + + if(ctx->instance->param->ssl_connection) + { + snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri); + curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L); + } + else + { + snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri); + } + curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); + curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb); + curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); + if(ctx->cb.header_cb != NULL) + { + curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb); + curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); + } + curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put_length); + curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_data_request_send_cb); + curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx); + + curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); + curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); + curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); + + if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl)) + { + assert(0);return -2; + } + ctx->transfering = 1; + return 0; +} + +static size_t curl_put_evbuf_request_send_cb(void *ptr, size_t size, size_t count, void *userp) +{ + size_t len, space=size*count, send_len; + struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp; + + if(size==0 || count==0 || ctx->put_offset>=ctx->put_length) + { + return 0; + } + + len = ctx->put_length - ctx->put_offset; + if(len > space) + { + len = space; + } + + send_len = evbuffer_remove(ctx->put_evbuf, ptr, len); + assert(send_len>0); + ctx->put_offset += send_len; + + ctx->cb.read_process_cb(ctx->put_evbuf, ctx->put_offset, ctx->cb.userp); + return send_len; +} + +int doris_http_launch_put_request_evbuf(struct doris_http_ctx *ctx, const char *uri, struct evbuffer *evbuf, size_t data_len) +{ + char minio_url[2048]; + + assert(ctx->curl == NULL); + if(NULL == (ctx->curl=curl_easy_init())) + { + assert(0);return -1; + } + ctx->put_evbuf = evbuf; + ctx->put_length = data_len; + ctx->put_offset = 0; + + if(ctx->instance->param->ssl_connection) + { + snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri); + curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L); + } + else + { + snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri); + } + curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); + curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb); + curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); + if(ctx->cb.header_cb != NULL) + { + curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb); + curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); + } + curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put_length); + curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_evbuf_request_send_cb); + curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx); + + curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); + curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); + curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); + + if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl)) + { + assert(0);return -2; } ctx->transfering = 1; return 0; diff --git a/client/doris_client_transfer.h b/client/doris_client_transfer.h index 8cfa4d2..324c9aa 100644 --- a/client/doris_client_transfer.h +++ b/client/doris_client_transfer.h @@ -49,9 +49,13 @@ void doris_http_ctx_destroy(struct doris_http_ctx *ctx); void doris_http_ctx_reset(struct doris_http_ctx *ctx, struct doris_http_callback *cb); void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header); +void doris_http_ctx_add_header_kvstr(struct doris_http_ctx *ctx, const char *headername, const char *value); +void doris_http_ctx_add_header_kvint(struct doris_http_ctx *ctx, const char *headername, u_int64_t value); int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri); int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, const char *data, size_t data_len); +int doris_http_launch_put_request_data(struct doris_http_ctx *ctx, const char *uri, char *data, size_t data_len); +int doris_http_launch_put_request_evbuf(struct doris_http_ctx *ctx, const char *uri, struct evbuffer *evbuf, size_t data_len); long long caculate_http_sessions_sum(const struct doris_http_instance *instance); diff --git a/include/doris_client.h b/include/doris_client.h deleted file mode 100644 index 08f3884..0000000 --- a/include/doris_client.h +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef __DORIS_CLIENT_H__ -#define __DORIS_CLIENT_H__ - -#include -#include - -enum FSSTAT_DORIS_FILED_ITEMS -{ - DRS_FS_FILED_REQ_FAIL=0, - DRS_FS_FILED_REQ_META, - DRS_FS_FILED_RES_META, - DRS_FS_FILED_RES_NOMETA, - DRS_FS_FILED_REQ_FILES, - DRS_FS_FILED_RES_FILES, - DRS_FS_FILED_RES_FRAGS, - DRS_FS_FILED_RES_FRAGERR, - DRS_FS_FILED_RES_BYTES, - DRS_FS_FILED_RES_VERERR, - - DRS_FS_FILED_BACKUP1_REQ, - DRS_FS_FILED_BACKUP2_REQ, - - FSSTAT_DORIS_FILED_NUM, -}; - -enum FSSTAT_DORIS_STATUS_ITEMS -{ - DRS_FS_STAT_MST_CNN_SRV=0, - DRS_FS_STAT_MST_FAIL_SRV, - DRS_FS_STAT_BCK1_CNN_SRV, - DRS_FS_STAT_BCK1_FAIL_SRV, - DRS_FS_STAT_BCK2_CNN_SRV, - DRS_FS_STAT_BCK2_FAIL_SRV, - DRS_FS_STAT_MEMORY_USED, - DRS_FS_STAT_HTTP_SESSIONS, - - FSSTAT_DORIS_STATUS_NUM, -}; - -struct doris_statistics -{ - long long field[FSSTAT_DORIS_FILED_NUM]; - long long status[FSSTAT_DORIS_STATUS_NUM]; -}; - -struct doris_arguments -{ - char bizname[32]; - int64_t current_version; //ǰѻȡϵ°汾ţһ汾ȡ - int32_t judian_id; -}; - -struct tablemeta -{ - const char *tablename; //ʽmaat䣻ʽΪļ - const char *filename; //ʱҪļ - const char *userregion; - size_t size; - u_int32_t cfgnum; -}; - -struct doris_instance; -struct doris_callbacks -{ - void *userdata; - void (*version_start)(struct doris_instance *instance, cJSON *meta, void *userdata); //meta汾ڶЧ - void (*cfgfile_start)(struct doris_instance *instance, const struct tablemeta *meta, const char *unused, void *userdata); - void (*cfgfile_update)(struct doris_instance *instance, const char *data, size_t len, void *userdata); - void (*cfgfile_finish)(struct doris_instance *instance, const char *md5, void *userdata); - void (*version_error)(struct doris_instance *instance, void *userdata); //ļʧܣð汾Ҫع - void (*version_finish)(struct doris_instance *instance, void *userdata); -}; - -struct doris_parameter *doris_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog); -struct doris_instance *doris_instance_new(struct doris_parameter *param, struct event_base *worker_evbase, - struct doris_callbacks *cbs, struct doris_arguments *args, void *runtimelog); - -#endif - diff --git a/include/doris_consumer_client.h b/include/doris_consumer_client.h new file mode 100644 index 0000000..f772626 --- /dev/null +++ b/include/doris_consumer_client.h @@ -0,0 +1,86 @@ +#ifndef __DORIS_CONSUMER_CLIENT_H__ +#define __DORIS_CONSUMER_CLIENT_H__ + +#include +#include + +enum FSSTAT_DORIS_FILED_ITEMS +{ + DRS_FS_FILED_REQ_FAIL=0, + DRS_FS_FILED_REQ_META, + DRS_FS_FILED_RES_META, + DRS_FS_FILED_RES_NOMETA, + DRS_FS_FILED_REQ_FILES, + DRS_FS_FILED_RES_FILES, + DRS_FS_FILED_RES_FRAGS, + DRS_FS_FILED_RES_FRAGERR, + DRS_FS_FILED_RES_BYTES, + DRS_FS_FILED_RES_VERERR, + + DRS_FS_FILED_BACKUP1_REQ, + DRS_FS_FILED_BACKUP2_REQ, + + FSSTAT_DORIS_FILED_NUM, +}; + +enum FSSTAT_DORIS_STATUS_ITEMS +{ + DRS_FS_STAT_MST_CNN_SRV=0, + DRS_FS_STAT_MST_FAIL_SRV, + DRS_FS_STAT_BCK1_CNN_SRV, + DRS_FS_STAT_BCK1_FAIL_SRV, + DRS_FS_STAT_BCK2_CNN_SRV, + DRS_FS_STAT_BCK2_FAIL_SRV, + DRS_FS_STAT_MEMORY_USED, + DRS_FS_STAT_HTTP_SESSIONS, + + FSSTAT_DORIS_STATUS_NUM, +}; + +struct doris_csum_statistics +{ + long long field[FSSTAT_DORIS_FILED_NUM]; + long long status[FSSTAT_DORIS_STATUS_NUM]; +}; + +struct doris_arguments +{ + char bizname[32]; + int64_t current_version; //ǰѻȡϵ°汾ţһ汾ȡ + int32_t judian_id; +}; + +struct tablemeta +{ + const char *tablename; //ʽmaat䣻ʽΪļ + const char *filename; //ʱҪļ + const char *userregion; + size_t size; + u_int32_t cfgnum; +}; + +struct doris_csum_param; +struct doris_csum_instance; +struct doris_callbacks +{ + void *userdata; + void (*version_start)(struct doris_csum_instance *instance, cJSON *meta, void *userdata); //meta汾ڶЧ + void (*cfgfile_start)(struct doris_csum_instance *instance, const struct tablemeta *meta, const char *unused, void *userdata); + void (*cfgfile_update)(struct doris_csum_instance *instance, const char *data, size_t len, void *userdata); + void (*cfgfile_finish)(struct doris_csum_instance *instance, const char *md5, void *userdata); + void (*version_error)(struct doris_csum_instance *instance, void *userdata); //ļʧܣð汾Ҫع + void (*version_finish)(struct doris_csum_instance *instance, void *userdata); + void (*version_updated)(struct doris_csum_instance *instance, void *userdata); //ʱû +}; + +struct doris_csum_param *doris_csum_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog); +void doris_csum_parameter_destroy(struct doris_csum_param *param); +u_int32_t doris_csum_param_get_refernces(struct doris_csum_param *param); + +struct doris_csum_instance *doris_csum_instance_new(struct doris_csum_param *param, struct event_base *worker_evbase, + struct doris_callbacks *cbs, struct doris_arguments *args, void *runtimelog); +struct doris_csum_param *doris_csum_instance_get_param(struct doris_csum_instance *instance); +void doris_csum_instance_destroy(struct doris_csum_instance *instance); + +#endif + diff --git a/include/doris_producer_client.h b/include/doris_producer_client.h new file mode 100644 index 0000000..877f7a3 --- /dev/null +++ b/include/doris_producer_client.h @@ -0,0 +1,80 @@ +#ifndef __DORIS_PRODUCER_CLIENT_H__ +#define __DORIS_PRODUCER_CLIENT_H__ + +enum FSSTAT_DRS_PROD_FILED_ITEMS +{ + DRS_FSPRD_FILED_VERSTART=0, + DRS_FSPRD_FILED_VEREND, + DRS_FSPRD_FILED_VERCANCEL, + DRS_FSPRD_FILED_FILEFRAG, + DRS_FSPRD_FILED_FILEONCE, + + FSSTAT_DORIS_PRD_FILED_NUM, +}; + +enum FSSTAT_DRS_PROD_STATUS_ITEMS +{ + DRS_FSPRD_STAT_CNNED_SERVERS=0, + DRS_FSPRD_STAT_FAILED_SERVERS, + DRS_FSPRD_STAT_MEMORY_USED, + DRS_FSPRD_STAT_HTTP_SESSIONS, + DRS_FSPRD_STAT_REQ_SESSIONS, + + FSSTAT_DORIS_PRD_STATUS_NUM, +}; + +struct doris_prod_statistics +{ + long long field[FSSTAT_DORIS_PRD_FILED_NUM]; + long long status[FSSTAT_DORIS_PRD_STATUS_NUM]; +}; + +struct doris_prod_param; +struct doris_prod_instance; +struct doris_upload_ctx; + +struct doris_prod_param *doris_prod_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog); +struct doris_prod_instance *doris_prod_instance_new(struct doris_prod_param *param, struct event_base *worker_evbase, void *runtimelog); + +enum PROD_VERSTART_RES +{ + VERSTART_RES_OK=0, + VERSTART_RES_BUSY, //Ӧ300serverstart;汾 + VERSTART_RES_ERROR, //ȡӦӦ400ҪôǷҪôδstartİ汾 + VERSTART_CURL_ERROR, //curlԭδȡӦ +}; + +/*cfgtype: 1-ȫ2-*/ +struct doris_upload_ctx *doris_prod_upload_ctx_new(struct doris_prod_instance *instance,const char *business, int32_t cfgtype); +void doris_prod_upload_ctx_destroy(struct doris_upload_ctx *ctx); + +int32_t doris_prod_version_start(struct doris_upload_ctx *ctx); +int32_t doris_prod_version_start_with_cb(struct doris_upload_ctx *ctx, + void (*verstart_cb)(enum PROD_VERSTART_RES result, const char *body, void *userdata), void *userdata); + +enum PROD_VEROP_RES +{ + VERSIONOP_RES_OK=0, + VERSIONOP_RES_ERROR, + VERSIONOP_CURL_ERROR, +}; + +struct table_meta +{ + const char *tablename; //ʽmaat䣻ʽΪļ + const char *filename; //ʱҪļ + const char *userregion; + const char *md5; + u_int32_t cfgnum; +}; + +int32_t doris_prod_upload_once_with_cb(struct doris_upload_ctx *ctx, char *data, size_t size, + struct table_meta *meta, void (*upfrag_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata); +int32_t doris_prod_upload_frag_with_cb(struct doris_upload_ctx *ctx, char *data, size_t size, size_t offset, + bool last, struct table_meta *meta, void (*upfrag_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata); + +int32_t doris_prod_version_end(struct doris_upload_ctx *ctx, void (*verend_cb)(enum PROD_VEROP_RES result, int64_t version, void *userdata), void *userdata); +int32_t doris_prod_version_cancel(struct doris_upload_ctx *ctx, void (*vercancel_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata); + +#endif + diff --git a/monitor/monitor_expamle.png b/monitor/monitor_expamle.png new file mode 100644 index 0000000000000000000000000000000000000000..af398cc5ca25716aff7655509aa5887402df718e GIT binary patch literal 35516 zcmcG#cU)83)-{YGA}CEjniP#nlOjm(A_1g#r1v7dcOoEy5J04fROukS_ee)tL^`2J zFQEq#k`VafIrlv8z4v?neBS*7va{G_t-0nNd#*Xh_@Jq-NJh#?Ns@s^{^OdLw&Dwd%CY-9_%Ao?<<#T|2x<~;Usw|1zmvEtz4js?An*C} zdu_n2#D;(%MoamnoUXswJ`UtfJJlg`xePjF^$VG9h2ug{$dJR`R@1}Eo1c<0lisBY zrx^9d%4ixH>3+N(1EGAGe*tE^CP&3D7bd$dLQfb*Pp0)9cvqI63arege9hR{DX7B5 zC1}~jh4%PtF~|-%lARtB`e=?lU2tp1DB-b~p#7LwRmavq`qrG_)oCGL$cN68l&W6_ z5R4GMxBq8*`_8}dypRva6oMYUit4-oT>`#>Fhd5HIxUp{=cqq>v@A-OSC^V{oQb`R#{|ymM{-Wn=&sz(5;G^sU>Y}g`V9j6JL_G{gAyCLJpFG zS|^EQSQ+b#1fTnikXe3eB>@qq8S=3FlC z#GCg;#JfzCl7E}<#+`r0L?GC$WT0dpFD<#;7-46akTOzP6tOP_WyyZ^H$9KmdoEdDM2a$cb>6^ z{?J9X+dL>TjISBN+!Xh14wNk2?=cmS{ZCI2QHkdHoKwZy z7m;>G!+Z_mEip!ceA)wDQ{M0?K?je~XS*#wm4y(VbM1DTew`?_Kp$hFiq9fT%)qf( zxnf>#or!sd=kZd0YC0lWhh>G>UZ>;n7*Bf61T$T*P5ssifVm@qX@m+qG#={#1olG= z3T~;+IUOJA1ghoRHA`~;)gxIpw*RrR9z7>e_9=zF>MzlE3cEpXpDJF66rkdhw6zzgMnJOuXFDzh(a} z+HR2DTz`rd7%Bvu$a#~i4;QGAGG#ANX&3OMe?6Ux@4k8Gt=kGeIb=2;Zzi$`0<~)g zh9oWXXnFMm2sQpT_8|Ega?2Fbj0Rc05 z$M|>c;ScwYm44hY@~NI67mGvh^Z#P7EmwnT zdYeP}W68hoMQ`-aD6Ws(0%#l*1XZzUG)Y%~X=K?9vt*y*VRRWw>XrhmMJDE_iZCFA zD_2qA3o4#UmtsVRTD_{tNs76z(18>ak=yo64!`!psQ3$?LfaF@=Ia`cW>l|QH`gZ6 zK}S!C|GwROmjCf1yhjJoSW)=YKiAyYdsZ3n%FbY-3&~>5f-2Of#`_px*M5DQuN{N_^rDgVTEERTEXOQ1UntzZ&GEy5N`h( zTHrrJ`yLSmQg?8;ndHf_yLBV<^7}Y#Nk3Dm!@G~c*`fe;sjz~$gixB{#y;aeF9rN| zN9Z5Z|NWidQb6=CGyQXl((V6Whm3j#`Eanb|3u$)-Pd19|LfdsIz9c8JRvX`7;B+KQ&gTy zz^2ptULjOkwpBs?!QE>gY2PZEvDkD5(_e3C;rD*{aRnHE10qQPi>9*tbo-;Qm{^bP zBU#`%W(I40!@s2V1?{^;21RD^vJyhpw`|(Pkvm1?@dV!5#6jsmW}qBWJIZ5DmDyR& zPK($+H%**B;#4!r_@mgQoJpe&vG&KT$uP4bZDJ(?@e=v&ueszQ1mQqt(f)k$wc0E$ zZe2P0bgjv+`7z>0*KR%Z(fsVX`}>wGyH9nRy`HRk)*whPFS0&(Kt@V!F(-P6Tq!$!b?+$8bY z;pe;jq_t_{;9Q2GgMlIl9+(I#NX5$W~VpA^1~;`Lk__f{X6OuG{Av?XKs2r{1r28m3nC zr_oi#oUL0S-^?IiyY&hWc6@oThCGD>gtE-NL|r1BhvU9n;ttdG=Y_&C!N%F0pGn_T zIA-`MCBSq4&9&Ibidgbq)M)s>NepDk`H_q|ea-czw$X*}xmKlTr-k;v{O(Q$(Q5Od z2#x7)$x8D3JZLXYH$i9P*tbR#$Lh(N~2&-_7M;ob00dPCzRoJFeC- zrp(YO6Ihc*npR~ok)I4{UJSe_roq<8>>;=eaXLgvAN~VrCFpg z1&#_5rJBK()}(YQ6S|u$B58e%Ni{7oJB=wcFgKniB>x61VOe>6ae9ppvJ%P{(Vg0s z)WB(e5Rdx>kJMvO=u__9%|foBwINS6Z+P-*LpV%ESdA5*!3`@M7R9eF7)EFv)RLx& zzwU%^&BN5j(M>H|`|Jid+6kSnM!@x@9^K|EgXy(k*qIKOiK!gbPC$RKin@gPcYVHR zEZil+SHjcsK}%UQb(?EAMKd6ONbEyY4dbMkrGf|Hx2`%q&C4eLw~e!}S#5qvs+++WP&2MNqX*xbyxS>_AE~t+Ki&#} zp|rAbnTnaxEs*rH2bbqJhN>8#4C%M9S1F{n_zCj8>mDx7{r;hvG?b+z6#I?QY)#BF*v0 zm>pcr&pFq1jgIVx#uaBDU0@Hj>^s}frjgrIUddO4J80=$0_A&i&Yh@cVYgzcW?!Zz8zIQW!Be=vYIk? zHA?oX2<=Zc@7fU+CQb1K^dn5e>F?hED`(mgGRRyD+kkYkWABF04Ar8fs*XBA!8_~8y z9gUy;>C<(5`vb=$Xj*o(7(MdE-Qf@?PK|lc`i8`WXCY3}Chy{Ym5VOh5^B@s0OGP) zquLeoit#~5zos@G6}uQ10`{e7GI}eDyF>IJG));NJe!F(km1w)8|1@|Zt2jM{(&uq zC)e)GqB!5=oVUtG4$sn+o~_>QBi22iO$L(z7O&~u_oPSf@^K=6H)T0ebXrQezG4I- zBfL%^5qZmXELUb(UX*Q?`G;a3%e>zuuB_i~Tqk(56Fg-8Q>jMWBkyyRM9fwJ2an}C zg?TNAU$g$0XsF7&(X>X<)jK$Sm5TM)QhU4J6t1F8Kxg9>CJ*hwX4aQZR_5?*f`?85xocH@5njG6P_^;5JKrjQEYzmmV6x<<#tG?EPLs9~#KqUIL!^4w{nMwT z=HAb69eK4C4ETweo6WA(2u+iOrGdJ9?_YRKf7n&mvfL1ZyA`cS(pH$fk-lOoq=EH{ zr%Jlu5H#5LhL`UDidn8(ez*mg9fA~mv|5ezAcPlRTm`T99^}(xoHGkw`Ej(?%3})Q zRkK}6#C@r(UYOQu)GY=}H77H*bH>BUH@rxHWFi2drfr9I{>1O;k66@^N|G z*BDF4D7~zZf2F8cguFq7|wh#K2^RPGad%akyaniP?@d2%f%tFaq76FxZ%jX zfa{1xp~Q4XY^Ay5MAwJQ#cZXZh=BKTy5F+S%I`@Ia_4Uc!cSKCqWL1?^QLTd`3EEz zKTOr`U5goFcjJ-l^%`}X{JL0{NX@dlPF{SM@~aL2o7v7f_uzL^sRbaY{L+3bjDi9X zJ8NHl;#}3pEX+6j2Bj=rMHBsD|4~NMk(mCPTU`Li`=>a;)!R`gqFf|`PkWCXi8?2Z zaixZdjGJ?V6XPnA)=hYVCq>6@CuRXA48qBfJdeqQ8*#67<*EW|^i^^urR%uVA z<%8Y9c)If~j``}PFG#f^)O|it%k?!HAt3ds=8CJCHL|^!wW;VebkSnU3NGy>cCp-` zXt-FwrhV7gTPM@#i{-lej5c#`z{s0MC%tCR#P~pdV94X`nxx>M#q*P86HS>##=so}5OSzb5pN?%~Oh&hGH#ej% z8@D7zVY>X-w-fh-8zI$-@*VQ%eguN!kP3+_*?pt~o&U)ww&RrVRxRwQ{t|0wY5x*j z=D$N0(`(2i*|(Gt47Tk!lw2+;7mk?~BKh*ja=p1B`EW7bbwQyeHY*7#tj~_R*u=1y z)}#U%hsy+jL$e6u;h1vdbZkf36raxIE&_?I?FpygKpveve;CNBAp9yMig~Rs+wzd7 zjFBOYE`0;_HbPUkUC+pbFJ|LY2_-Z}3kA+Mu{Y?T zSVE&QO(_*R20w6(EDdwtix%EY+#yUEarNQ z(trS9V}|u%S-Iw8M2}MA3p(l6BvPloE;g8jb8sT5pCNM`^C_*F4-uSl)n?_jFerAh zIe0Q1dlnV(_kx|0{@j^@JlOvkE%WA&YjK5y5>?bBf8Iy$tT8Z~9- zi3I25!4+5!ie^`5GRY<}_9j)v^k=p4OafaJ}zS z!88OF#YG=?qPKL^n@S_Py>*C{U#pS=Vy6#2V*2??j>0_IE!J?mAM*EPO&NJ=^FQQi zR7V6IU(kA${;u)M+~R|+I0vC1>f^+{d`oDjDQ#6%SkODt*?6;4RQG_07!J1~!hzX_ zZ9>q|-3ohBF{s{YYC!_B@Vhbd?7MSvM_y5!;&3};Gwoi8n>g)L>YO#nzRg7iqKSq4 z#JnjhQ+VF&NE=71`bF8kOU&i>%w;qD3Pwk_erkB1h8cb%y7j!L^v0fMSrVyV?bQrx z;q^;@D%eu{NFy;*);Kw^1XF%qzDQ%b?pzg97X1dqIVe;Dbu%=_~Z zrA&uCgnJPsiaEE=cGh(E&VN6!k7a))cU?l?f0w^6+trsB*V++A%0g$^%g5;pVFzCv zbB|#|H;K5a6}vCPN3kxBb9*YUIAL}w`Q-v@cdM58f8AN0v@Doo&tg0p=NRU^lAbo3 zKJ6~_)4Bw8SRA&QSU$?R`oUGKAt;#{?@Cj3CK$7b;NhZ#)5BQ%SQ3$n-_m9e-8f1k zc>0-u_m+TYsOUi9xWodk)}nNR&pLKidT}>Gp`2o~PTE1kxciOajx!zUlO=^jW5eaU zKC_xVn5;Tx&d(e~;n+D}t22)n>F)WyHn3a3(v?)+;yK5bTMH{SdtJKwyF^L;B;fT| za@I>tuAad25EUw*x_|2aK=}YtL?wT&=PoeY9_H04wVxZ#jb1}(p9V}HI0y0WPUY~| zx)-CJ77Wp3z4DUQ1x(8WvF2Cy!Sp|**L3uBtMxMLH}g>i{GSdF^*hyTHY7^f4lvM7 zx-`P&tD75qDL_@8426J%9HHXlfHb$_?b}u%7o4;U`z8W)N9iuk8GyzC{Q56n{D|?M zFLU2mHy-KK}f}Oh#LF7mQeym>yUyU!% zV%kDGbc~kc01;ufq@=dEnivfj}Ut8KbMA|#66acT{cBN-j<*v5aO||L3rQXH_6lBjt$ik0s zWA4GTLt`dRP}?oSWt$MUEWs}>@KF6aR%Ms(d$t4mw^{N{&?n^{ux&~wz}8FeV_x3{ z?5VOZ{#`|#BVC(3>ZZQ6pJgA#bExILa}66pSA+a8+ciZ@^UY$p2F#`#$a+n{>SZ4P ziSJNaV}9a5=DvfoLOQ5^QSUZ~`TUbD8Q*?a$aZ&ImP*xhqqdFQH{3$E-Y`ensFSKS zb~s$D&S9N)*`4=CYe){2U3~B)hIici{lk9malct)0l0&%reekNa}(KYjZgMN6M??l z0A|ZO0I&W1B?Wzm%NXYSJnt%#mOuA~^pY)KaQJRmvqSv z8uc0zK!;*$M6wb`%S}&uA9tJY=3Zd4_HVIX+rpHyC@cCG(F4o=it-h?NO}fIxhT&T zrj!DFx35@eB}J!1j9hySeHUz@(0IT;j;1Qs>-A)wi?gZGlTMuujq%4#Ixly*Tl%9V zwmeG}Xf+rQ*_&Qu8l01CY*z1g@s2O6SG&a4R~Wiy(Ew{Z8+R#b%NOypR;W^2W^Vmp zBUZX2$Hi+?0i$ z)$;Xnf+0H0dQi(=phBSPhBAO2=;k@QVEFzK6OVf zN5bK=Sn3J1K;fUs&^)-f*Jf%pY>#;%K1TksXIRy zw^jDP^&=J!Ejdc%6={8b6NcU81Fr*^=DgW6ZE*Sv=AW3s_gpN97ep2n$5|J8Mhe2U zmWJy?qexwUPkd(TgkFZuk~o+7DOX{2V$@LW2`PLqc)X#$ccIXiZ>z~(wP_Vh!C%^4 zk5GGA^8}5c_p6WXX);FBJm~s* zqd(2BjX?sw4{qh)YUk2~g>=4(`hSyor6palSKm-B9mCUbKs|`XVwiRL(b8P!U5g`k zxC|e3zPt!0%`xOwYwN?ke!oZ)BR$TGyZc4#yizRh195PrQ5{6Z)h=g+SLXw$6(vwH zw#;mI?hkYF9VMmvd@YZ6*PZK{M6%Uk$0ye3$4Ajpp)#yw{gZvl3Le&#sm@KX^`4xCc*VW1ADiIDem`FlQTR-e8r~i%t&slqJ!Z({vjG2 zu9n`N^b2C%z!n|WH}M@bUUDjcMvB_zvvE8!kdYRYgVu(Q^Xp~L7kBdaM^PfWlU=C_ zq`4;NNQ=Jw%ALrK3xSdZZCz(4k)~$s@2qlG#P^TbgQkl#L!s+}PT~}_bjm_mPvYE7 zddy}5L7ldP{dHT0Rp!00Xc_&%rJO0cnCbh8;T#(D3>3!kaG~q+H|#vRmC{5};B`i* z_UTeaVe9)Xfr^?}g70{rLc&g-d_nDT*R!lmP%jJqDvyxjs`#>xMRO>vg@Woz-m>yJ z+b1Z)rbfsC6)J^eq7O&eBNL5%v1!&f>V9D$j!jn=}DmkVGR!;H1s77r&Nk z6w$aV7T@b@YQ&iqbt`u^mOI{gg4lRq_<>wvtfVZ2;=u0nRSxpBanZG?hmAbtheulA z#H|rXthvLd>nlwhcQ-6Qf$9?kU+^>a46B`+PP@%oLX!KJ%b76BuMSLcYBr2R;_vD1g;qHz= z7NApSFK32Ha}HGJvXFIwZp%Ew{Sap_30@K17;PPBoh)u~KD0GL!xJcFSS;XcN;|bV zHMD+h%ftB>C&jkv=M6hz_5nc@I=trD&mlj8p(N%#mtmd$afxFT_h*G4YaG;h95Va} zkyU5|S;Z67OB?qeH#P)|8FLRZw=B$|Hka2sRGps1(Yu*0`CK&Yxj|2L@xD#fIb0u4 zBX{zRIT@8j^-BrZIg*~tzAy98nN@NX+cbWQ#VYhLfL*kH%N`1;n+uk9f-h9j)kr@^ zvNG+!mp!CGmVMOySO!bbrCE$>JS#AecHjOGq@7p;{#u03`P2fyCadI{CbM+OE4Kkt ze^vLwzVB$zzdkajF}+Ncn044I?2b~ItEEAK+cyHZ$~}N7v!5apoiZ;EZ&OIE9!>ozQuP3iA`{WthB~8-Agdtbj)~99%pcQ-tNA!vn-=G!jS&(wX~7N0Go4>Z zb#Wbg*M|5yv+`0F$2RnuajSPLPn_xZ9-*G@w{;Myj@U|D&b@9h*t-dFr07eAeY=cGOgx-2 zVkeW}z*r+1;>M$8tR@O<(bXFPgz{^33tD`p??11kAxc-%uKPe57g>;Ai-$rIJxZKx z>p8~u^eyw~rUWf(1dfczr}}XsVM!sP)S|5+9y0RA_i=_R@q5y8-hjH@MfS(9S)ba3 z@N1Wl-gUD~C~X2%^8e~wcoWY5i8!&}_Q_m5ouS$soFUDg7V>HnaFY}_T#rRA_Cr)yipH|QNtzREDdY{U`i@=hh~j)=u05iCJ{f;w zc(n?rHWak&v-4?mq|1*JNdvj~CVtL%&ohLj3Dcz$%egFSCMt*44#D$v+QvBg@`m)B z8Q^hL$^DYX=I+uk(0*emWpATp{{=pv%cj$g(v|KpIkP+F6si4=NbuIze0) zx)7*wZkh3)N0P#SRgYNoDETOf(nyAVidnGhOib*1?s=>nGnc09SJJJ-e*a|6axof) z)Y0u+=e|;J7;j%Mye z1mkV8Nyd{r+#_s2|KqZ=@yp5FWsg|YF*ib>#(F@L+`Q;f@#iR?;qvkJi_S!-VVVo> zG3|o+y16h&Y6qcM^oX9}#~}l!89c~(WQ&{{U(@`=Y`NZn&ElJax)#kA1l_+lNkcl_ zdF-=|XSxKk{lriM6}x1B!g0qS^Wtqk%rk+LXQfxhU3T8&*ww3B zI!|yIT>9uVHc`>SD2H3i;fH8gfy^Je!0%gkwCO*(+t@VF^3DSQP@O&D&N~4%>u?&F zB`)L4(b{I6BByA(XYVb1WpOOnr=pzVI5DpK4i`PtFrWE$KVKT}%peA<>eNZ%JU^O| zIox(K-1Z~KR#YwI11UNr{&P1iMa_VJwZRJ5ef7+1>Wdmkc0v9iP4|4zMOhIDoDqYx z$EY;|b%y}lg|Z&GlV5d7O}fe6g~i{Nso3eJTjj4bW8@I%-1d~ZN}q%oo2+Tz~`anHREUV zV}pYeG(4+^0A~PzHLt|Skq;qw#CBk;Ur_;Fxy>{oDsWOGZVx+EO=;t)f`sv8@X6rt zkyaGts@hhHY0k_RAdp2niA3K3?WezK8%sLkXyy0>>$JjyANAU0IS}^ zJC0$@@bwP&cPINQO|A_Q1dpfVD?52p>9&XK1A9c!*@>fjf}azLJ)ugkdoRO>oZO@i zNc9&G#A)||(r=bfY35})ZZq%%?aoOEwP}&wJD*C*l4O)YW5v>956c>0wV*m&qbZwO zskM8sE$d9Ce;O3@sNfl+XDo9)txG`I38@YBqnZZGrn^2ERD9N-(0^WE#b}ZyUJ%idq%a)-aj`?v5^Ls)M zfjY<5p>1(ShD^sBqlGwWGD^em49Jat0pYgy{{+5G5ujG*` z<;s1hq=w;z5LqhVeEuEt4p}0^`tL(u#%IW=F%A`^Tj&SVwu;ocLsjj#tl`oVs-(?; z$ZdV6IT(@=_0Bua$Yc{-`y7mO&D9fs@rJ`9qSSI0$PaCH&C=D?@?Gi_0 zy0m$L;d)Kq((45sBu)@dv#Xeutrl>)FPhRcUSl^T1RwoHf3oL1yw_U)<$ z$jS4DH!eHh=6Qn5-EbMpf=9%|b6%OYH`lTg@%1GyX(xHQz5`R>cnoh4L|3=w)5$(z zwpk3%9B_QBg9Rr8drH_N|5(EbNe^7kqSb0?$0EDNNDK6M~<5_B#fZb=tMCRF-D8iS!q(=0^zd~qp$DyC}% z`M^(egRyTj%pV3w&`Pe<>y2+#R8}W0qv|axnsXSw&BTb&q<6ouxxh)&R+i^&cf=vD zwYF90+yc-oD-r#=@*gb*b2`JEwLfU7=n&?85-+&iI) zQm52uZ~O3&BG!f37=)GrY^lIHg_t-6azYZ;A*-Mjp`nZo5~ifcL_l9Of4>*#6-&Dq zN`1|xC7HB$p9%a^_tGR&{39cya`#~3j81Q-8#SB{RcGzZX)zdNxG=GIpLPMsrXU=B z_}rcY1ooyF80FI`a28V7qpFTdwjfqLX;mGiddqz%>K%ucP1 z`LPbYYHIo}OU2uy`p~v-nL+h6E}rWLamj8jN%<<-by#n$Dc+&1ADwM6wlX-;IR9-m zM8IB~X;)qaO?rP12l6tMZD#q)jQ{ddXV1Jv-?Nw+>$z^B;l;3O8s1V{YkJOQ!B1x^ z0y~OiWb7P#Uxs&`aHa@xV*_EG&Sra<84?wqBMMJ_2)^$T>3uaocaE47_9>n)J4pQa z@P^5OGwNLGatM#Z2R;X4n2T@8euA)>=zyl_&uQs#)_Z@~(DctPX$hT4qhFFV~opB&7;NVIRA#SHHX_(3{&qAQGE0k(3gQ*qbe z5*L>d>gGR!90WJaEIT0h3(|(B;aQy>FynBbKg$+ zw#^j$(ibCxy%@u3>$KV1P~xeoDNjiNbIZ#dg|knlYhG?S2j;jWt84A0au>|d*+vj1e>U3$_;kV;k?tYxy{=dg>2_ z3_z%;#Jxi&Zq7#2){9%PulHQ6m{3~I^BkS7r>)a*53Lu^}*6ro1FCt^Bj8> zJ6&iPlJ*owOIvfv668R-K_?@tHBG!Y92}e1bZl#M=s2qsX7=L+3j17|6*i#jS{QC6 z+|itA1&vjS50+KXa1q!Pnc)#Wyv#A4m<@c5bQyx!vD3z=JULMjA4$;ey6-E?WiDJE z#$&#a$)5244&{s~Y8gRAN(f88COrk)SfOg54^0Ed4WrOfEa=k=j+04P=izgrLdpAM`3sS5 zub>jjiHQc)FnHcRDz927#3g1*kJE4$RAnQ}IM4c2d=@dcZqcFtGup3%T)N z@2HRAlT{tUhj|Nq+8yV2>@kQ1`vyYfFT+17IH+B}E7^hj`#(86g($H%zIvoj~ z`OZeCmP2WsFxpCvhNb2GWWLH~i%ccDLy0sgd)_}a`vF6Wb6W{TOPhi;Jor7c5`cfx zW=-9-SIt>^8Hud-N(RX|$#|rzOCQEgcZMye=`g-_ewTQGo~iIMpH4v$`QE2%iChLH zs~JSTonR{1X9*%ZHd*3b8t!ErwuY~Z!dpD>z8S2P2_?7fx)hV4 z>dp*Q*Ya0LhfLrm&V9JeLeUMCnp&UdcBU~ty;4yeyJC$AQl_@?FD^W&`wBYmwkWp` z*jEf&pA5h_63mVC+EL%8H81s{jp63Lh%(W%khpKGRS!b4_s9Y1;VsGRZZd@!#17z| zKeDPFS7?0~>+f-B$+$xXfQ+SjAQ1hwrV2TNZ=exBoJO(SnC`(Uy79pcbr8=RZ_F1D z&M9RH+F_Uq0I_&23#MxSk{!F3$oZ@OLTnc*ld7Re&yXe_Oc?cFjQ6kvJQSIe>ONz> z>)ze2T11}D7KDyyZr-c@NlLAUc=z@%3OwnetcZkj#Dm;F6!>PsW^a!dOcMNep8p}( z|6rE26W92B0U-pGP z0$-sXe7gJl&Z#!B-A9&P_uB$d;?XK}=ui2vv=b38n3d#wwTT1MUoo@E`GebCAYPn? z9t<0_PSi`(+H`+bN0VPEWPg?r4){-AGYYnLShseep-v+b;bmF`Cbm^1e)^q{Poh!+ zi?aS}I{z004B263cw&Zv6JF8;!V3H&H~H)HUFKhMjQ`ZG;PUWYsek{UDnNqk|0Oy3 zORmC#V>Y|^D+~GL3%*GmzDYr;m!^6T=x-2WLjAwG^ia6mInH%^OOBWBNLG79vdMua6(_Z6 zCv6E&3yU3%@K$C@JJB8NRu#vJxT8Os^%Pw@uXbuJho1ps42U4o!^?P3S45k6xq& z<(+NpN@OFMO#QRoh|X)`c)i-u_kP;&8#paiEfqb@G_wCON7>KY;Upo8WM}TV_X59W z<9~J@?3R&svC;oT_1t!03U#oKCJrI=#tVslhsAN)&K=3lbeXj=_Tn+eEKk6AaR&mBUa|v_2e&)lMBS$y;JO7zw^^g z=Wk2vh+5nUm<4a!>guJ?UYDa4Z&Q0l(=3nY*necX>xX+cC&8>3&`qidp z%IRw(6mKEPP?Edy61JD?&C2k{DVg2uf&7B@MDdUZFM09VmZd|ci@L|RI8Lf^Sr=L| z%aB47rImqPUTV-nPTH#Kfq;EOru*OWPncdZ*`#*4nNT+wkCCaAA`L~(s*$@p;&N?t z@kto;ORnXhaPgpkpg&q#vXoR&XvzkSxk8il(Nh90Tk=>$AO+a$tElVW=2XQc?Ql z-~*xsQ+LqZNSHm}Vh%`qk~|}JC^T+Q{`quokC+M=|5{D0!`@*lE5`8}`(KmT^&9)i zUP^l(P~ieYdu4fhLxb&E)7@dIHW^|K0zYcxGFnWQFQ1Rs$bH(di|%UTOmbg7*j$p% z$(=v>5K%S9@BG)?8|+%0UVd zB?reZ(ngq@b=6}R!SP|24%~G+rCV%B_=otj$6`87Kc5Lh!vwDC*~Yb?ADH3;Fhz)# zjl+~5V;b*1{CQASsw6SEE0a7ZC@8^-anSn|caEi_R~MhxPvwLmoCVH>)jgL81X3H% zWV+CAZb5#(OPtaBmAFJ*@5D=CH%s^VwQ)&dQ<5)KxEMM+tu9qz><}o8l>j|)aox|? zfw_tlnHrC)N)6-`i&=ZUj$%=(D@)MQP$V1O9L_RP8+GpSbBEQ{*P$OWBZ3(`85%w@ z&N;Dgh%EQap4{BscFD8NKR+F>-?L!#ndB1+?Z^@I6bnjy z$Ld4;wmy5eSDdh=ZxF+ip!{uFAtze4{mmNsLi@I zO-wgRzbVaDrdk@#!b%YDHf7of;1gqb8UC*Y-~2~{)Lnl)gherW)(qFM*TBrygL1UPPPMUx(zJ# zL>S>jlRnl&vS07>UM%GMz)j`U49=UdB9rcrHs=K+@%4FWw_VEf1p;| z2VYljy_kdUKqY{65XA{T4dnR7@Zm(x%(tzmuJQ31Y8SqRyQ~OyBl~;PKHBnV`o__8 z`8IWt38T)Crvb^kgC!O30y zo;j!v)+wc57*bH4ZMGpC{YBVlsiS?6_|~W?cInrh!~d^8AQ(#o?PvAolTzDD{bhsNU!O z#s;)gl9KmbVG4D+mKjwp+9X9Zh({*o3&9~Kp-@Q<8saA;Gg(CSZMa)fQd+7>X%gA@ zT<0{L?8GP5XXu+8B=$`KIqoBw{OTH@kwOw~yt>}1f4komy~W4U z!FgY&@9$Vn&YMacDwVQbt*ZSAmkRCfbp)oDz^lfrbuN#o#_gHnKnB6U(>9gjoeNte z1#-DKEyw{jU|zSy>>9DJLd_ko3Ckw>W$kJH`$}3NqgWh)NV1pmQJID4EH|PX)BBNJ zOTrT}koF2?f9H=jE$LR(F1w$cAyNp(`8L}Rr)%_A9*GvtB1o4SMW>_jZKtiVx+cK` zm&hj8tXxxgFUM2qA*rtIhHH2K1RSN9DG5jcbU8>r(D^nlD^w9Qw~labemm z<(9Xf{yfJj3t!;BILAJk8kt}szthTso4Z-$-@bJvoqr3UV(7A5Qiy$CIT3NlbpMcb z*Lv1Dzc@Soa({-J87_g)nDYiNm3_hC$zaR4PTDUu3d+7LuQ)=^@pV9|> z%vhaVFZ|2?lsv>i1{s2xV1EgMuUwfafbtWzla|lMxAvNIn8zo!KntpLZ$<-TE&6F!RWD-Jn5Ch;JdxsRD5-rDi9?q)xoQ*=3{z_?*_zWI;3PmH68 zu6zmJCv`TCmhT4$GdMG`OS=)Is}5swwu6OiB7ZLl)~+kR5g+L6Wr8%@o7)8~vJ
    ?8>Euez52uM*nh|~Z9kKi8+y z2|;eHiuDO_>EIBF;*xh8q6|(GZnI( zv3WC>B{n)QdoI!2GCdcl@Yt|CUh;)i#iJwDV)iyNu7mHd(KCXdnSA5x?zw39S>ej8 z;^!*a>Fd5P+BQy*!Y%n;(~dHSj9jSHdb>1cU!|-xo^r$H%4DtWYf998hG9;w&qcON zsXXWCTFp|;IuLaoKI%-NKGsDE-EGgT6;v$ek$L!CEZuffM&wgF#VUR>dA_JmW{Ep% zjqHPwh^VOC9@~v2O>c0jm)VVbuB9XP9YfYV(&0C=z-NqSB6f;IIMOPSUAy*VCl zAp3&Bv)Vv6Sp0?!h($W2qL;MjAk6GBzlvzkq9z>fz zKY|^hu9;jJBE&sQDi%g}gm-E#3uG`Xi^sc9&0&Tahd<)`BcM-nTfIB_+JHa>vI`#s zR~ND@Za72x<;Axn*gKLN9`glmIEmwiLJCPQHwV%E*xj0hJ$-o_JMr?wd1~?wR_2xE z>ej{HaP*jOu?eNHu+U(ww{ z7SLv)BJ;q07nXzY#jY1^x1w<&0Dav5$fW{SF1v}7(GHYFLOtv>2#QH>e;KLhe z=O)L7?RtaRHWVw?*yRm0?|3u0h1Rg9RL2cZCbG%MaPIa?MviW#!wigbKw;5^LAHTs zuYZ3(td(`{>}q^ccCxjm3+Zs4{D|zsO7?qJp-A}FV+LW8Y~qjE&n_w`lACT33nG~L zRt5?6QOk(TJ+vlOWwVmCenFZtqt)<&M6>VZsStlfj09Ffv}e0Jna@U`fZ8tQjd?-I z$PhD)*IVV}xN7A1x^Fxa*b*z%vTr%2jR&>xOFW=g@^ccd?6(|8 zED7f+)GWU=ROL)^0PCLIYv&ENuv3|Qtmw?wl_51VsD!|dEPLG|C=cROR47Sn<9BhzXOy34m zKVR6U=r(bW-o#r&s}1ZCn-xE;#H3gPbQgjkXo4ruOn-9O-AcOiTWDsQ)9Nfz5tnzF z_UeKX5Ykc})Z1s|8li^T0TV{LXd`Vw8l{|uiD8MPx8cPH^!aX)XG8qRo$j%qL2Wp~ z`P1ady)H95X})<@@+DQW!GvOPHqs_cVVAet!c@@JZD;^q*h&uW5nb~)G2_&!(oLHA z{+5+ovB~aY@ysWrx6cf@FtL_99!ApHxNr$R=$Zf8O$|eU6l_Gl|gN?ENL;TlC^W1YP*Kws)>ZrdkA<`ngtkHS&FoaK*r4A@WPa)l>WZ zY0jbzD|uRJ=~KkQki7H;Gao0s{n70dI}KNzjU$Ue2l&IUs#QQYwkiX7u75puqD!m( zSv@JdN|%e{sb#uZ$I^U!&S^IWiu*Sn84i zGZp771K#r7@sW{Ca-ZIE+N*OaXLxI{^Th8I`Rak;-nWjJL|kTLN%^$gxcb9^Ugo{oi! zPY$++yVgZ07kW;(`eAOd>@RotfltXF6q@ro5(zoRbM)pk{qXC=@QUuQz)QKmf}Y#M zaaRf5f5)d4zD~A-$Q>(s8W=gV9=5Ha`auzIG)eMgD_?(ymlK{T$qDg=o+g4Tx=C{j zLgT*UHGaX)&t@(O#wfX%l&)M)k_Tpx(GBp7@A*9t4^JmkuI6hM8qvf#LiRB$q|S*C z7Q@zAYjL66eMl<@-e~pI{Ci`KoOHQsrWn@8YubU2kWA=`<}V|7#HGV~fCwUW@r}gR z(Ec1}C6WNYMrQZ8XA4N}$#@k9Ukp1pLTSrXGp75(VqZLQte3Qra!I2KIU3_iAqh0y9UbYzps6O;bo`M+qHw{K`3e(??^j6W;52eYJZIIM09 z1CKDA{u)r85psSavZT<59*dUtoUlo~>I4F!+YHAyQK3rPRY#VUk1B^CWs;w8Tf@uU zBFp8=l@yKe@EHq##m?@F{*F3v0{Z-M_fBx%itrBQ=q}yuaI=tUw3Enr6<;$bFTGxF z!P?ZOtae(F=`trAwC2u`Q^YrxJf@33skd{B0D5L(73(0MU3H;m>b?4f1zU5IS%ig=G;?vCi4}SDuc6$9-x z8Xmx=C@ywOzMpK@cD#XsCro(rjaf@@v>vS_OdUGd?spH3QO`2KOe*Vd$ zko5hr0AHD!n*D-pkSohjcuL>!SdVH~VveTPZn5Libf*n5)s?7C1yx-o-;}brfL@8! z*toVC63e57BqfIEB1r)TEVD0^hhPzF&E2y1@|Yxd`6!_z3%C%;>U{Sv!=ykZ&vjrE z=xCo=;WBSChxn8mpCoyaNkEW8?`=(Q+)*PN8IAr(&`JevgGynvxcFg>#jswmtcPP0 z@WJ|Cu`*{YciSq)ZFDvaYgKb<;0E6FvFAgkBycEgeCl}6cPnSQZ&S0G+0L;5IXmAL zN{pzHUy6s;=Hp(fNTb#b)`wLqA6z%oS%^3Mu04JO4X}rQ8)b5w1X9We3^s%A}hW3Qm69Zlu7d0Z!Q0nl&*ZcuVsT z={KDpllJm#(jYu8rSKAEbG>%ZZIM)XYuz28f%f#K$dpab7LgGm506YDb}Cv-iC1U7 z-);A?!M0HPr=c#(+~=-NRD{eKZFpA3wPX}v&r-7`a?SWnj(@11A0Ej^;thz=K%-{t z_?i`#(4k9ple@BH%n7okL09y?O33$H#lZ=zEw`cwv+wtIt>1e2vYr)$gBsi!+|6vx zIP^Mn%$f_k`CRxT(dc4s zRv&8DE6M^6VH;ynD)E~9{NAovD%d4!@!K^U@3V3Ad=oIS?Dv;xj#S`H^KkgmD7 zA)k`2Nf!e8(tbV^zBJ2kn|i$Fk-ePo(5-=Em;B5HXWwcrl@X)b3y~c%SrKnD%Ehvn(*+rEa-7dBE0Hpm9v|8r6Hf`fd9$b z%fCdq|0$xJ|L`qsnA#sC;A6r+66pRew{>62|8~3J7p`5CkY5HE_VvupVZXhs{D9we zzJfUv!i2zF*+*4PvscprfFS=TYT%eItHJHsc%}+cANjNaC#8}*a0+?=S)QNE_uK2U;bYK4o#c%a;p&&D}J8S#<$Sa7pm&qwMA z;iJD`(qP@T5(Lvta!SBdav=gL>D+LpLoduO$@r&N-Vr>-DcdaQ^fM*7&9Lubi)eatJy z8jq>*i@JKCnCeWRXL+MHI$P(K&low=A6b z++6OOxLw+Hzr^~WQbN(AA%~ol&KFAP7&KRU2ejxj8Hh>ZnWoI#eyv9OOoHU6fd+?l z)k$B;#=8dY#+Y9<2{TBgc5S4hZC=^fMlrm0wOJ*=;#pH#yzv18GZ>d1S|tD z@3S&AF#ukWY@>NVdHvbK@o40m0!QFamW5M^x7KUaqzAlTxpmJqJQO&R zLJHne;kE>sypP@(!;(cIF816><8HMggz-1lOHy>%Zngyb0KPw}l$-^yA1bAk>MtU8 zyy7JzvX2~*LB(^7gZ&uAPNlWQG{{(k+)}ogUFfk6w%YehuuZMm^?v%?Rocy2$kw!+ zbcJ_vb>d8|ZRgtC!8*rRep2mI)ImYm%%)I0uYBw=X7f?+m9>CVyjIIDZWmK?k2) zy%Bf0Zg2WAqJ@ilG4k>_4`L;6F})}Ac)aE21|9-|^8w=$c=r1ohhu8)h|Hobnkjd} zb?}13`f5=qsB-qX_@umf8Oyvg1GO6SU0ranfBiVrq%tVJe)_R|tf-wQixD43_JqE2 ztCrNB@lm~j>%Bf9BS{Qp+v22}rpkky-t+9eN%gHJwdWWo2MuJfyJH3>9UPd_Yt3(J}5QbpB@-%v}?IX6V_UTptS08{2~HvI%|C@wW1ACmwU6 zRE~Be`rUai!$Ww{!LjILA0x8o@2!E(I~TnSkv%Cr&)E5)gmU#)MT3J$^3{QvqY>(9 z-F(z+$J2Y2G~b8L2*hgXgogb#G%IlSs~Fc#qKfnID~a@EOmf0uw%s12*<8vx>B`Y(&f}L@7AC4FYLad|7_17lkbQ|{cYLr@i z#&K3Aljzmp%|&}`)^Hf{6dV>RXSc^_GXIvm9I;t>!Kq}P<01&o@aoW?yfr7hm11R8 z)p79cEsb>^uaPaFmb~Xh$M)3^SZO~v zCOXC;u+|^r#GQK3?Ub`&%}r||^$(>1nV!O~Img-g^t3vq06g?Dhvk7J4n4D4w({tv zHbrxoQ1|$l2B&yNazC8wjjOrZ7QeJhb^}w$PKG`Kd?Df)o&{f9csh);9F`(bRYq=V z`-*>y$$qf4EIz@*L+FhWja)GiQ-HV69>Oy*kLrnS^+g`v%brP`$tlasi8ho_DHk?R zncfsJ?3Gg}f!=#(yBMK_V&T!S5c6fTcc}$+pU&dkia_u9AfXA$7$fR3Wz{f=X#43g zx&>Wz2I<=gPknD!Q3enC@v*SuHql_;A7Y?O@R#M*_3Jq1NSc~yB)x~YmE zo$7}s+7$b3+=`;#_baGR)}T}6(Q$-H82OA*%ii1uV|Xm`(92hU@)c^$8Ydc-Yv`SA z6@9Fkc2A}5)@>WR^3!h~kC8%QS5I7`rM}}c<~O}tZsW1qnv@ocTgeOtIbB5dJ`-vD z^*_>G@;A+5qo${$&o%DYwc|ZfquKS;CFH=C(x{u(GSDP*bv>=XNVD#LpudFWp(ZB~ z5J9A;xVYPAFaAcAjrHpM9_N(;Tb=w9Zoi4C!vCc9BAm69pn1k=Qsn!+e?2o7W5-6R_AMKN z3&NI%W40|0rlkqr`wUnk|4a}4N3nt@Lz{ywPP9CVDM?Pj)oQ8pkk`RuJfDkSouGTn zM4DFid2xm6VyzNw?=;D~=*FJz(C>5(6z%ApxprUCd9Tw$`?%)(khYDU;3Up#;<)+D z;aa402ZHfjgy;1}6_Iu5qPX=X^Uyjq>2aYCFh}FZgcfR!)U)CU>4>4e@Yjx5S>B9t zSV>;s@QwSfDUtwV9w!CPnQjDbC}C#-rQ)6$QdHSWC#(Br>NjQ{s_5+(w-ZPX`1A3P zyOfpUNv`TYREub^jzD;yQ5JsoX!(q9!Z4iQbI%8STu8?-<-mPqzs;kqfO6q$dOUo(1cPu;sDSNw6TA*B^>1AN#2|Q(Ca5!$jXJaw2bidOQ)y!nwY)-L7Q>~%VRVf?F zM6|NJrJor|7QWs(vN-FKGGHb_FO_0eQo1kyC-08~056tWQuXWC5!=7^n4YX3!4WC@mHjsVl#&Pl*mpGsf%`_3T(Fbe~QEvQ{#E zd)u7t!yjp4U&^Dy5S z!yWKClk{)uv}KE1M3OR9ELpNqsf$ zIlElOQ9iMW#cv_*V7{ce2wuZ*Me>x(HEr?~i?zDY<|88=@kxsSvHfn}<6e%u?(M0F zt?CJcf!M)O1^X7v2K!m_tIUyVmEt-Wsg2TPs9#{NH6K%uzZ$OP@g%ORGdz(X`9OKQ z3X|MR#NP>ZhMbXmRZtl??CH<-n6iTL z+-eYxf!=$xuozJ-mRc9C!HVb0>wa5nOH|DPf#)ARMnhZ@V1?1)D+UfhQ*^^>>V4Hk zCh2F!ZllY<_BnfHjqT(QJGm9MKEijeRKEX$;$|}zn-+l)8aKDZqxM8yGam~)Sth_Ub*+G{AT|8ANkq;wT$ul z*Y@c@CYME1kMew~&Ro>B>*; z|CPtsh4mk6JN5T8#Yp1R9`nB-nSVht|8GJve@_A_d~9rs8XJOrt`F)lt5N|0-to`R z*yn;5rO>ueG(@^HGl@vZ)KQNSYnF2sf4wrcdAvNy0xh(ug-a;v8_%g8E;ANDzBn?8 zay-iEu$VrtUa23E7xRAjk-c8Z8dr~7*}%$xNY7>zxhz4j$%7fEfo@QD$yW3#yg6;d zAE)qmzy>l|`~KQ>%|>2NB1>L&ps>P|y=-r;h^Td?rOm)o)naZyFox4=Z|tRyZKzki z&v7qu$44G!)hy+7iGvWCx^<0QxN-~V^Zf7*r&G{(21iiN>JyGqcW!0?#QI>l)6;>cs*_kxf#*l$Mp}#7LVKWwuXwzG&&cSYevD z@N{>z%3GRghmfDYqi4l(o3{R&r{RyFljQ#NRnKeOXj| zoVWe8_<1z3Mq?n$kHP^TmeCVoUig$z6cilpwP^#__$SZ{&BGm6*FPWzhi!!gy3XTV zPHrk2#i?t8qW$*5A9^L)8BIxSV~=*ELl{teSvzn*z;=-9Nly9hz4oZ@uMCWiDB0Vp zWP`>0%04ouO-d})Br7LPiWNQ*10E|1WF)=qwKgFX=5y6BfYA8KW$ z1nkiBmn;U+S_-8=vXGvRC}x#&V*4h+0jbh@*6g(d;PT<|?mjp-@6Bp7b(a%3TFv2X zid=jw=W<#&B||AV>Yu_gjmVpS8kX5cfKVkhj?!D>zPaPH+vR4q&r?1p7W->#aiw*m zEtZL_>Nsd0aLQoYhGp}B1zx!eg)#9%r$1fPx4-on)ILWMHxs4@B0 zBHQ#HU}RPA!}{OH^h{aZ;^7fR{H3Z1V5tqeNqqlpWDj@h(Cp5pBDL)*rs92NoLO2M z45@CLRUq26cu-D4rDWov$f(5JnH7vlqn4<~K319$O7NDaZYoeV8}l|R<4WOyhV66BZ9 zBkpBuC!P;5f+yI4MK_K%?EYXxwZ(CM^aME+O-FjQLFLUhoiA)}i_43&&Py3eDH6x6 z1w{6zIHgZ@`ejUEd4eA{|AfhW-C0xN##r-&o{Va!aGn9|6YYR~f&s8kJUOPpN`Ayx z&S|L3LTGhN71*9@n)q*{W$uE>`pz*Pw7R_+^UJ?k9%)#cv*wlz zMy-FrU%V(s!uah3f)Vp~QY+853Gl?s>E zuS6%VX;{ACy}{gXupLM;?450A>InmX6uTLt?J0GPP5lI38k8m{onvi!93=~bpRa5a zXC5e=On67Am5s;1y2A9}A zF{-xL`5}hR_Q!)SdUnB&_tl1d`&rSX51-!|PGj6aeiC6&ODBE@apjHc%b;Iun8Zgf zM>=28H%@+jR$l5mInZ0Vghhew9|Xt@T|UN4JSOn#{~5St&?r>0+&9j;GD~WvERF8W zUc_46L0u(>Uo#Yk%7)#T##c^UP|L~8OJI-X<>ur#^m}!-H>uQ2D1qs#;uHkd6wDz< zz0r>ggN5zVrZX~xRbQ-(KFIo7(L_>s`x1)r!E`;?8l078SV^|IjbTc14GYzEVv!vA zfyo{(faZ=@c2cxNEo#jp)Hs#F95rLeg+tJ|W=Y9$Z0V*w98bhuwz!r z+h4Q;BDuTL(-3kQSUM{1*cGbzDjfUr2aQs&v)0zAwbsMa@Vvj8{-SW>$eYD?Qn1ai z0H8^@0h|pbhE&;n)Oqdn8?yg9AQQ}U3}K<0vC!78Ax2?-AuH27b6D5=HJss_*&(6C zeOW|8C}gIq=o|Is`PuZ8>UY@voNZBWlv3+?>3=o;;Ngelm{8c)zx~Shn+Q zCWp6ZHc-RP*ve#zH>R_YUsHd6pu1O0r*bQBh45+IY^amA(R ze(o@q5OvLg>r@c{M>(~4s^Hy$uZaWFOJ%PPn3l2(p_d^m$fsMl_@KonJlUcBl|@K_ z>AB>f=pTz*_4QlpO@3V^S-YLY?Y_faDcidZaek7E3t$Pkn!vs6)Gqe1kE6n0Wt`vH zV&*Z{En~>BjqeXLT;-)~3P-vY5wQav5Yq;Chj$LuYyfa|?F0{1OO!avtwUE}-)b+~f54($B}C^lnFD~#O&|C&F;tg?}y zZVpYvxN0#ct7)VtlprtTHUTkt^Vk*7V!`EtTA?Q=?ohAoy&N!d1&7W~TJAb$167v> zdGdhqv0(KSZN+w?vC)P92~v(jTuQkJB>I>teO_@t*5D>DcT#GvuDPWX?-meWDUWuN zuT>$!JLnO&O3aR6Nu2i#QX0~;piZetwd|)-35#2-xOS{+Q zy3Otm(hjR|yK?2FaxzXLJpmW+qNU!60?oOR%Q=aUZuWoF(Sp3gQdp7K^U7KmWHb$dXwT=3nNi)A|A z`QPIwFYmI(`T7&^&JjbRgHQ1yVmv&&G_?x_%UeIQC2J7}spfsOx7DoG@L55V3blM>Jl)x+-o0lUt}GSd8}^w8=_%lfTUkMHrVyA6(i zVM^0wZ?6+!bmp|>&az~xNwT+ZFxUpm+(9{cjT355^9U%<@hm)QZrNvRjrc63gWs@Lj1zWc~wnYGV%lV0Y=k^ z#4Msw#+9By3&;}RjdxXEq)nHLwIrLJA(ieB15X>c^RqAOY@11*!1IfFY@V!0%*TwoFk8usb+MW*Z>as~n)d>@8Iyw`x-*P3lRs|*&t9b4X*Fy*kfC>zH}oW`PH95aKN0J z@IUqgM{ z8#%ufJxaGYSs?vgj?Q^q5N$7ie4cMlc zictDRm*H-V_6gQrEv#pRg-ATFW<|uH{r=V|QHU(bdx;)!5o*uFGRa$ zEk?9RC(M9&K&bnqKL?W-n}5&f#Xg@DT_3;+zXoDR8HAG@3j7PH0`8@|C1gA;03#8C zLbW3*&Qq`VySJj3n(~W=g-W)whJ$VG>ot8G-q}{iS@b~#0Kt`4)OPydytAgC-MqbW zK;$hIL#o#?V6Y4sv7v)zd}lv|>?W;x^qwYKI!{OK!bZoPR&TBAi|@IAX$17Z#Zy$; zymORmVsK}f-8A-fgmpwQ5xHI8G1plJ)YL=@F=3jGt-bdiOb2a@M|=#BTt{wp^6xJ?KY%Cx?Os zb1k^U9<=i(QkeoG8$Ojn)mhot#QYs+1AymFG{j(!C8_9jKH%0HkQB+-u?m&+v@)JA z?nd_S=3GcgXLxem;$q_UWIV_-rh}mXa7#*$n@=j1&+?VIR=AuIwOP{9^#ja{JL}}` zZIVzq)*l?z3v4|^2#$+lPWMx6rOwQG_r|Hrc^4$?aUf)JGrdFKwftj7#I-6(Skg+PmL3oess5Hj;P582NB`5_u;>0@pC3Npi{onLC&Gei3 zsl!6_8|J=JqtXJEO^e3y>x%y3-I#fro*hV+`mXL~57wz_2Y-UQdqTOpR}%I4&M+E? zl$j}KU#!2SBx#qVqVk{PL^^6hg`CpTyw77&D^bj~f6?hCJ>p3j6uwv9U-*AmN4D2A zmXI{IHDHdU36+~r>TiV7CNt2msbo46bZ5t&Zgn#H9*GxOP&^_NA}RIgHg>GC*cax3 zxkyz$%uoq^MjrN=f@V5H@OPT!a>JJN)DQ=(i>kN8^N3!gx#v|w z%112z282*G4R^9iH~TLjLT9&oh4$iWirhm!+mt=)@|D(lP&E4eAzNkO6QX{V;sBWK?*&@YV9Ra_>0$HBg9~a>ll>bxd@eicOzyb5Cf4Go-_#b8Z{ro5I z&I7@EKVP^c2*l+5eBrJk5PJ9Xh2P46djI*tFOZkF{z#YnfA^25a4Gr5(f1UN^7}^H TSAE_8c2pl|C>Gv-`s#lHXkc;{ literal 0 HcmV?d00001 diff --git a/monitor/配置分发网络运行状态-1626422695467.json b/monitor/配置分发网络运行状态-1626422695467.json index f8777aa..e28f18f 100644 --- a/monitor/配置分发网络运行状态-1626422695467.json +++ b/monitor/配置分发网络运行状态-1626422695467.json @@ -65,7 +65,7 @@ "scroll": true, "showHeader": true, "sort": { - "col": 13, + "col": 2, "desc": false }, "styles": [ @@ -218,7 +218,7 @@ "pattern": "Value #D", "thresholds": [], "type": "number", - "unit": "short" + "unit": "none" }, { "alias": "MAXfds", @@ -266,7 +266,7 @@ "pattern": "Value #G", "thresholds": [], "type": "number", - "unit": "short" + "unit": "none" }, { "alias": "Restart24H", @@ -337,7 +337,7 @@ "unit": "short" }, { - "alias": "LatestVersion", + "alias": "LatestVer-T1", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -353,7 +353,7 @@ "unit": "short" }, { - "alias": "TotalCfgNum", + "alias": "TotalCfgs-T1", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -361,12 +361,77 @@ "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, + "decimals": null, "mappingType": 1, "pattern": "Value #L", "thresholds": [], + "type": "number", + "unit": "none" + }, + { + "alias": "LatestVer-VoIP", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value #M", + "thresholds": [], + "type": "number", + "unit": "none" + }, + { + "alias": "PostServer", + "colorMode": "cell", + "colors": [ + "rgba(40, 233, 13, 0.9)", + "#cca300", + "#bf1b00" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #N", + "thresholds": [ + "1", + "2" + ], "type": "string", - "unit": "short" + "unit": "short", + "valueMaps": [ + { + "text": "OK", + "value": "0" + }, + { + "text": "uping", + "value": "1" + }, + { + "text": "down", + "value": "2" + } + ] + }, + { + "alias": "TotalCfgs-VoIP", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "Value #O", + "thresholds": [], + "type": "number", + "unit": "none" }, { "alias": "", @@ -449,14 +514,21 @@ "refId": "D" }, { - "expr": "0+latest_cfg_version{job=~\"doris\"}", + "expr": "0+http_post_server_status{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "N" + }, + { + "expr": "0+latest_cfg_version_T1_1{job=~\"doris\"}", "format": "table", "instant": true, "intervalFactor": 1, "refId": "K" }, { - "expr": "0+total_config_num{job=~\"doris\"}", + "expr": "0+total_config_num_T1_1{job=~\"doris\"}", "format": "table", "instant": true, "intervalFactor": 1, @@ -468,6 +540,20 @@ "instant": true, "intervalFactor": 1, "refId": "I" + }, + { + "expr": "0+latest_cfg_version_VoIP{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "M" + }, + { + "expr": "0+total_config_num_VoIP{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "O" } ], "title": "Doris运行状态", @@ -514,5 +600,5 @@ "timezone": "", "title": "配置分发网络运行状态", "uid": "HZfW8wi7k", - "version": 7 + "version": 14 } \ No newline at end of file diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 3f609ed..92077dc 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -4,7 +4,7 @@ add_definitions(-fPIC -Wall -g) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_CURRENT_SOURCE_DIR}/,,$(abspath $<))\"'") add_executable(doris ${NIRVANA_PLATFORM_SRC}) -target_link_libraries(doris doris_client_static libMesaMonitor libevent-static libevent-pthreads-static libcurl-static libevent-openssl-static openssl-ssl-static openssl-crypto-static cjson libLevelDB) +target_link_libraries(doris doris_client_static libMesaMonitor libevent-static libevent-pthreads-static libcurl-static libevent-openssl-static openssl-ssl-static openssl-crypto-static cjson) target_link_libraries(doris MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat2 pthread z dl) set_target_properties(doris PROPERTIES CLEAN_DIRECT_OUTPUT 1) target_include_directories(doris PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) diff --git a/server/bin/conf/doris_client.conf b/server/bin/conf/doris_client.conf deleted file mode 100644 index 062aeeb..0000000 --- a/server/bin/conf/doris_client.conf +++ /dev/null @@ -1,35 +0,0 @@ -[DORIS_CLIENT] -fetch_fail_retry_interval=5 -fetch_fragmet_size=5242880 -fetch_confile_max_tries=3 - -fsstat_log_appname=DorisClient -fsstat_log_filepath=./log/doris_client.fs -fsstat_log_interval=2 -fsstat_log_print_mode=1 -fsstat_log_dst_ip=192.168.10.90 -fsstat_log_dst_port=8125 - -[DORIS_CLIENT.master_server] -max_connection_per_host=1 -max_cnnt_pipeline_num=10 -https_connection_on=0 -max_curl_session_num=10 - -http_server_listen_port=9897 -http_server_manage_port=9897 -http_server_ip_list=192.168.10.8 - -[DORIS_CLIENT.backup1_server] -max_connection_per_host=1 -max_cnnt_pipeline_num=10 -https_connection_on=0 -max_curl_session_num=10 - -http_server_listen_port=9897 -http_server_manage_port=9897 -http_server_ip_list=192.168.11.241 - -[DORIS_CLIENT.backup2_server] - - diff --git a/server/bin/conf/doris_client_csum.conf b/server/bin/conf/doris_client_csum.conf new file mode 100644 index 0000000..7055fc3 --- /dev/null +++ b/server/bin/conf/doris_client_csum.conf @@ -0,0 +1,30 @@ +[DORIS_CLIENT] +fetch_fail_retry_interval=5 +fetch_fragmet_size=5242880 +fetch_confile_max_tries=3 + +fsstat_log_appname=DrsCsmClient +fsstat_log_filepath=./log/doris_client_csm.fs +fsstat_log_interval=2 +fsstat_log_print_mode=1 +fsstat_log_dst_ip=192.168.10.90 +fsstat_log_dst_port=8125 + +[DORIS_CLIENT.master_server] +https_connection_on=1 +http_server_listen_port=9898 +http_server_manage_port=2233 +http_server_ip_list=192.168.10.8 + +[DORIS_CLIENT.backup1_server] +https_connection_on=1 +http_server_listen_port=9898 +http_server_manage_port=2233 +http_server_ip_list=192.168.11.241 + +[DORIS_CLIENT.backup2_server] +https_connection_on=1 +http_server_listen_port=9898 +http_server_manage_port=2233 +http_server_ip_list=192.168.11.242 + diff --git a/server/bin/conf/doris_client_sync_t1.conf b/server/bin/conf/doris_client_sync_t1.conf new file mode 100644 index 0000000..717ce9a --- /dev/null +++ b/server/bin/conf/doris_client_sync_t1.conf @@ -0,0 +1,28 @@ +[DORIS_CLIENT] +fetch_fail_retry_interval=5 +fetch_fragmet_size=5242880 +fetch_confile_max_tries=3 + +upload_fragmet_size=5242880 +master_slave_sync_on=1 + +fsstat_log_appname=DrsPrdClient +fsstat_log_filepath=./log/doris_sync_t1c.fs +fsstat_log_filepath_p=./log/doris_sync_t1p.fs +fsstat_log_interval=2 +fsstat_log_print_mode=1 +fsstat_log_dst_ip=192.168.10.90 +fsstat_log_dst_port=8125 + +[DORIS_CLIENT.master_server] +https_connection_on=1 +http_server_listen_port=9898 +http_server_manage_port=2233 +http_server_ip_list=192.168.10.9 + +[DORIS_CLIENT.produce] +https_connection_on=1 +http_server_listen_port=9800 +http_server_manage_port=2233 +http_server_ip_list=192.168.10.9 + diff --git a/server/bin/conf/doris_client_sync_voip.conf b/server/bin/conf/doris_client_sync_voip.conf new file mode 100644 index 0000000..5600721 --- /dev/null +++ b/server/bin/conf/doris_client_sync_voip.conf @@ -0,0 +1,28 @@ +[DORIS_CLIENT] +fetch_fail_retry_interval=5 +fetch_fragmet_size=5242880 +fetch_confile_max_tries=3 + +upload_fragmet_size=5242880 +master_slave_sync_on=1 + +fsstat_log_appname=DrsPrdClient +fsstat_log_filepath=./log/doris_sync_voipc.fs +fsstat_log_filepath_p=./log/doris_sync_voipp.fs +fsstat_log_interval=2 +fsstat_log_print_mode=1 +fsstat_log_dst_ip=192.168.10.90 +fsstat_log_dst_port=8125 + +[DORIS_CLIENT.master_server] +https_connection_on=1 +http_server_listen_port=9898 +http_server_manage_port=2233 +http_server_ip_list=192.168.10.9 + +[DORIS_CLIENT.produce] +https_connection_on=1 +http_server_listen_port=9801 +http_server_manage_port=2233 +http_server_ip_list=192.168.10.9 + diff --git a/server/bin/conf/doris_main.conf b/server/bin/conf/doris_main.conf index 1220cc5..fcf8a6c 100644 --- a/server/bin/conf/doris_main.conf +++ b/server/bin/conf/doris_main.conf @@ -1,13 +1,15 @@ [DORIS_SERVER] worker_thread_num=2 -server_listen_port=9898 -manage_listen_port=2233 +consumer_listen_port=9898 +manager_listen_port=2233 https_connection_on=1 -cache_file_frag_size=100 -#doris_server_role_on=1 +#cache_file_frag_size=67108864 #index_file_format_maat=0 +local_net_name=em1 +http_post_cluster_on=1 +#http_post_max_concurrence=100000 -business_system_list=T1_1;VoIP +business_system_list=VoIP;T1_1 run_log_dir=./log run_log_lv=20 @@ -19,20 +21,25 @@ fsstat_log_dst_ip=192.168.10.90 fsstat_log_dst_port=8125 [T1_1] -#1-Doris client; 2-local file -receive_config_way=2 +#1-Doris client; 2-local file; 3-HTTP post server +receive_config_way=1 grafana_monitor_status_id=3 +#producer_listen_port=9800 +#producer_concurrence_allowed=0 store_config_path=./doris_store_t1 receive_config_path_full=./doris_receive_t1/full/index receive_config_path_inc=./doris_receive_t1/inc/index -#doris_client_confile=./conf/doris_client.conf +doris_client_confile=./conf/doris_client_csum.conf +#doris_client_confile=./conf/doris_client_sync_t1.conf [VoIP] -receive_config_way=2 +receive_config_way=3 +producer_listen_port=9801 +producer_concurrence_allowed=1 mem_cache_max_versions=2 grafana_monitor_status_id=4 store_config_path=./doris_store_voip receive_config_path_full=./doris_receive_voip/full/index receive_config_path_inc=./doris_receive_voip/inc/index -#doris_client_confile=./conf/doris_client.conf +doris_client_confile=./conf/doris_client_sync_voip.conf diff --git a/server/doris_server_http.cpp b/server/doris_server_http.cpp index 6c2dad1..f8bdb4e 100644 --- a/server/doris_server_http.cpp +++ b/server/doris_server_http.cpp @@ -16,64 +16,11 @@ #include #include -#include - #include "doris_server_main.h" #include "doris_server_http.h" extern struct doris_global_info g_doris_server_info; -static inline void set_sockopt_keepalive(int sd, int keepidle, int keepintvl, int keepcnt) -{ - int keepalive = 1; - setsockopt(sd, SOL_SOCKET, SO_KEEPALIVE, (void*)&keepalive, sizeof(keepalive)); - setsockopt(sd, SOL_TCP, TCP_KEEPIDLE, (void*)&keepidle, sizeof(keepidle)); - setsockopt(sd, SOL_TCP, TCP_KEEPINTVL, (void*)&keepintvl, sizeof(keepintvl)); - setsockopt(sd, SOL_TCP, TCP_KEEPCNT, (void*)&keepcnt, sizeof(keepcnt)); -} - -static inline void set_listen_sockopt(int sd) -{ - if(g_doris_server_info.sock_recv_bufsize > 0) - { - setsockopt(sd, SOL_SOCKET, SO_RCVBUF, &g_doris_server_info.sock_recv_bufsize, sizeof(u_int32_t)); - } -} - -int doris_create_listen_socket(int bind_port) -{ - evutil_socket_t listener; - struct sockaddr_in sin; - - listener = socket(AF_INET, SOCK_STREAM, 0); - if(listener < 0) - { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "create socket error!\n"); - return -1; - } - set_sockopt_keepalive(listener, 300, 10, 2); - set_listen_sockopt(listener); - evutil_make_listen_socket_reuseable(listener); - - sin.sin_family = AF_INET; - sin.sin_addr.s_addr=htonl(INADDR_ANY); - sin.sin_port = htons(bind_port); - - if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0) - { - printf("bind socket to port: %d error: %s!\n", bind_port, strerror(errno)); - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "bind socket to port: %d error: %s!\n", bind_port, strerror(errno)); - assert(0);return -2; - } - if (listen(listener, 1024)<0) - { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "listen socket 1024 error!\n"); - return -3; - } - evutil_make_socket_nonblocking(listener); - return listener; -} - struct version_list_node *lookup_vernode_accord_version(struct version_list_handle *handle, int64_t version) { struct version_list_node *vernode; @@ -96,16 +43,49 @@ struct version_list_node *lookup_vernode_accord_version(struct version_list_hand return NULL; } +/*ֵ: + *304-ͻѴﵽ°汾ͬconsumer׼ + *300-δͬClientð汾*/ +static int32_t check_producer_ready_sync(struct doris_business *business, struct evhttp_request *req, int64_t cur_version) +{ + const char *client_version; + int64_t clientversion; + + if(NULL == (client_version=evhttp_find_header(evhttp_request_get_input_headers(req), "X-Doris-Sync-Current-Version"))) + { + return HTTP_NOTMODIFIED; + } + + /*request from sync client, check http posts-on-the-way first*/ + if(business->posts_on_the_way) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_DEBUG, "HttpProducer, posts-on-the-way: %d, meta response 300", business->posts_on_the_way); + return 300; + } + + /*Client汾? ˴ӻӻ£ClientʱӶƣڻȡ*/ + /*ȡʱõ300Ӧȡʱֱͬİ汾汾һ£304*/ + if((clientversion=atol(client_version)) > cur_version) + { + business_set_sync_peer_abnormal(business); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "HttpProducer, client version(%lu) is newer than server(%lu)", clientversion, cur_version); + return 300; + } + + business_resume_sync_peer_normal(business); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "HttpProducer, doris client is OK to sync for business: %s", business->bizname); + return HTTP_NOTMODIFIED; +} + void doris_http_server_meta_cb(struct evhttp_request *req, void *arg) { struct evkeyvalq params; - const char *version, *bizname; + const char *version; int64_t verlong; char *endptr=NULL, length[64]; struct version_list_node *vernode; struct evbuffer *evbuf; struct doris_business *business; - map::iterator iter; FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_META_REQ], 0, FS_OP_ADD, 1); if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) @@ -128,29 +108,21 @@ void doris_http_server_meta_cb(struct evhttp_request *req, void *arg) evhttp_send_error(req, HTTP_BADREQUEST, "Parameter version invalid"); return; } - if(NULL == (bizname = evhttp_find_header(¶ms, "business"))) + if(NULL == (business = lookup_bizstruct_from_name(¶ms))) { - FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); evhttp_clear_headers(¶ms); - evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no business found"); - return; - } - if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end()) - { - FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); - evhttp_clear_headers(¶ms); - evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, business invalid"); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameter business invalid"); return; } evhttp_clear_headers(¶ms); - business = iter->second; pthread_rwlock_rdlock(&business->rwlock); if(NULL == (vernode = lookup_vernode_accord_version(business->cfgver_head, verlong))) { + int code = check_producer_ready_sync(business, req, business->cfgver_head->latest_version); pthread_rwlock_unlock(&business->rwlock); FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_META_NONEW], 0, FS_OP_ADD, 1); - evhttp_send_error(req, HTTP_NOTMODIFIED, "No new configs found"); + evhttp_send_error(req, code, "No new configs found"); return; } evbuf = evbuffer_new(); @@ -210,24 +182,14 @@ struct evbuffer *evbuf_content_from_disk(struct table_list_node *tablenode, size return evbuf; } -void doris_response_file_range(struct evhttp_request *req, const char *bizname, const char *tablename, - int64_t verlong, size_t start, size_t end, bool range) +void doris_response_file_range(struct evhttp_request *req, struct doris_business *business, + const char *tablename, int64_t verlong, size_t start, size_t end, bool range) { struct version_list_node *vernode; struct table_list_node *tablenode; struct evbuffer *evbuf; char length[128]; size_t filesize, res_length; - struct doris_business *business; - map::iterator iter; - - if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end()) - { - FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); - evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, business invalid"); - return; - } - business = iter->second; pthread_rwlock_rdlock(&business->rwlock); if(NULL == (vernode = lookup_vernode_accord_version(business->cfgver_head, verlong))) @@ -266,24 +228,29 @@ void doris_response_file_range(struct evhttp_request *req, const char *bizname, assert(res_length == end + 1 - start); sprintf(length, "%lu", res_length); - FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_SEND_FILE_RES], FS_OP_ADD, 1); + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_FILES], 0, FS_OP_ADD, 1); FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_SEND_FILE_BYTES], 0, FS_OP_ADD, res_length); evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Length", length); + evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "application/stream"); + evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "keep-alive"); if(range) { sprintf(length, "bytes %lu-%lu/%lu", start, end, filesize); evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Range", length); + evhttp_send_reply(req, 206, "Partial Content", evbuf); + } + else + { + evhttp_send_reply(req, HTTP_OK, "OK", evbuf); } - evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "application/stream"); - evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "keep-alive"); - evhttp_send_reply(req, HTTP_OK, "OK", evbuf); evbuffer_free(evbuf); } void doris_http_server_file_cb(struct evhttp_request *req, void *arg) { struct evkeyvalq params; - const char *version, *tablename, *content_range, *bizname; + struct doris_business *business; + const char *version, *tablename, *content_range; int64_t verlong; char *endptr=NULL; size_t req_start=0, req_end=0; @@ -316,25 +283,22 @@ void doris_http_server_file_cb(struct evhttp_request *req, void *arg) FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); evhttp_send_error(req, HTTP_BADREQUEST, "Header Range invalid"); return; - } - if(NULL == (bizname = evhttp_find_header(¶ms, "business"))) + } + if(NULL == (business = lookup_bizstruct_from_name(¶ms))) { - FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); evhttp_clear_headers(¶ms); - evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no business found"); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameter business invalid"); return; } - doris_response_file_range(req, bizname, tablename, verlong, req_start, req_end, (content_range==NULL)?false:true); + doris_response_file_range(req, business, tablename, verlong, req_start, req_end, (content_range==NULL)?false:true); evhttp_clear_headers(¶ms); } void doris_http_server_version_cb(struct evhttp_request *req, void *arg) { struct evkeyvalq params; - const char *bizname; struct doris_business *business; - map::iterator iter; char verbuf[32]; if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) @@ -343,23 +307,13 @@ void doris_http_server_version_cb(struct evhttp_request *req, void *arg) evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid"); return; } - - if(NULL == (bizname = evhttp_find_header(¶ms, "business"))) + if(NULL == (business = lookup_bizstruct_from_name(¶ms))) { - FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); evhttp_clear_headers(¶ms); - evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no business found"); - return; - } - if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end()) - { - FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); - evhttp_clear_headers(¶ms); - evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, business invalid"); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameter business invalid"); return; } evhttp_clear_headers(¶ms); - business = iter->second; pthread_rwlock_rdlock(&business->rwlock); sprintf(verbuf, "%lu", business->cfgver_head->latest_version); @@ -464,13 +418,6 @@ SSL_CTX *doris_connections_create_ssl_ctx(void) return ssl_ctx; } -struct bufferevent *doris_https_bufferevent_cb(struct event_base *evabse, void *arg) -{ - SSL_CTX *ssl_instance = (SSL_CTX *)arg; - - return bufferevent_openssl_socket_new(evabse, -1, SSL_new(ssl_instance), BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_CLOSE_ON_FREE); -} - void* thread_doris_http_server(void *arg) { struct event_base *worker_evbase; @@ -492,9 +439,9 @@ void* thread_doris_http_server(void *arg) evhttp_set_gencb(worker_http, doris_http_server_generic_cb, NULL); evhttp_set_allowed_methods(worker_http, EVHTTP_REQ_GET|EVHTTP_REQ_HEAD); - if(evhttp_accept_socket(worker_http, g_doris_server_info.listener)) + if(evhttp_accept_socket(worker_http, g_doris_server_info.listener_csum)) { - printf("evhttp_accept_socket %d error!\n", g_doris_server_info.listener); + printf("evhttp_accept_socket %d error!\n", g_doris_server_info.listener_csum); assert(0); return NULL; } diff --git a/server/doris_server_http.h b/server/doris_server_http.h index 3092a55..934cb56 100644 --- a/server/doris_server_http.h +++ b/server/doris_server_http.h @@ -3,7 +3,6 @@ #include -int doris_create_listen_socket(int bind_port); SSL_CTX *doris_connections_create_ssl_ctx(void); void* thread_doris_http_server(void *arg); diff --git a/server/doris_server_main.cpp b/server/doris_server_main.cpp index c7703cf..2bc10f4 100644 --- a/server/doris_server_main.cpp +++ b/server/doris_server_main.cpp @@ -7,17 +7,24 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include -#include "doris_client.h" +#include "doris_consumer_client.h" +#include "doris_producer_client.h" #include "doris_server_main.h" #include "doris_server_http.h" struct doris_global_info g_doris_server_info; -static unsigned long doris_vesion_20210804=20210804L; +static unsigned long doris_version_20210825=20210825L; int doris_mkdir_according_path(const char * path) { @@ -54,6 +61,79 @@ int doris_mkdir_according_path(const char * path) return 0; } +int32_t get_ip_by_ifname(const char *ifname) +{ + int sockfd; + struct ifreq ifr; + int32_t ip; + + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (-1 == sockfd) + return INADDR_NONE; + + strcpy(ifr.ifr_name,ifname); + if (ioctl(sockfd, SIOCGIFADDR, &ifr) < 0) + { + close(sockfd); + return INADDR_NONE; + } + + ip = ((struct sockaddr_in*)&(ifr.ifr_addr))->sin_addr.s_addr; + close(sockfd); + return ip; +} + +static inline void set_sockopt_keepalive(int sd, int keepidle, int keepintvl, int keepcnt) +{ + int keepalive = 1; + setsockopt(sd, SOL_SOCKET, SO_KEEPALIVE, (void*)&keepalive, sizeof(keepalive)); + setsockopt(sd, SOL_TCP, TCP_KEEPIDLE, (void*)&keepidle, sizeof(keepidle)); + setsockopt(sd, SOL_TCP, TCP_KEEPINTVL, (void*)&keepintvl, sizeof(keepintvl)); + setsockopt(sd, SOL_TCP, TCP_KEEPCNT, (void*)&keepcnt, sizeof(keepcnt)); +} + +static inline void set_listen_sockopt(int sd) +{ + if(g_doris_server_info.sock_recv_bufsize > 0) + { + setsockopt(sd, SOL_SOCKET, SO_RCVBUF, &g_doris_server_info.sock_recv_bufsize, sizeof(u_int32_t)); + } +} + +int doris_create_listen_socket(int bind_port) +{ + evutil_socket_t listener; + struct sockaddr_in sin; + + listener = socket(AF_INET, SOCK_STREAM, 0); + if(listener < 0) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "create socket error!\n"); + return -1; + } + set_sockopt_keepalive(listener, 300, 10, 2); + set_listen_sockopt(listener); + evutil_make_listen_socket_reuseable(listener); + + sin.sin_family = AF_INET; + sin.sin_addr.s_addr=htonl(INADDR_ANY); + sin.sin_port = htons(bind_port); + + if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0) + { + printf("bind socket to port: %d error: %s!\n", bind_port, strerror(errno)); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "bind socket to port: %d error: %s!\n", bind_port, strerror(errno)); + assert(0);return -2; + } + if (listen(listener, 1024)<0) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "listen socket 1024 error!\n"); + return -3; + } + evutil_make_socket_nonblocking(listener); + return listener; +} + static int doris_chech_name_valid(const char *name) { size_t i, namelen=strlen(name); @@ -100,8 +180,11 @@ int32_t doris_read_profile_configs(const char *config_file) assert(0);return -1; } MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "cache_file_frag_size", &g_doris_server_info.cache_frag_size, 67108864); - MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "doris_server_role_on", &g_doris_server_info.server_role_sw, 1); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "http_post_max_frag_size", &g_doris_server_info.max_http_body_size, 67108864); MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "index_file_format_maat", &g_doris_server_info.idx_file_maat, 0); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "http_post_max_concurrence", &g_doris_server_info.max_concurrent_reqs, 100000); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "http_post_cluster_on", &g_doris_server_info.cluster_sync_mode, 0); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "http_post_op_expires", &g_doris_server_info.post_vernode_ttl, 1200); MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "scan_index_file_interval", &g_doris_server_info.scan_idx_interval, 10); MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "https_connection_on", &g_doris_server_info.ssl_conn_on, 0); @@ -122,18 +205,19 @@ int32_t doris_read_profile_configs(const char *config_file) MESA_load_profile_int_def(config_file, "DORIS_SERVER", "fsstat_log_dst_port", &g_doris_server_info.fsstat_dst_port, 8125); //LIBEVENT - MESA_load_profile_int_def(config_file, "DORIS_SERVER", "server_listen_port", &g_doris_server_info.server_port, 9898); - MESA_load_profile_int_def(config_file, "DORIS_SERVER", "manage_listen_port", &g_doris_server_info.manager_port, 2233); + MESA_load_profile_int_def(config_file, "DORIS_SERVER", "consumer_listen_port", &g_doris_server_info.consumer_port, 0); + MESA_load_profile_int_def(config_file, "DORIS_SERVER", "manager_listen_port", &g_doris_server_info.manager_port, 2233); MESA_load_profile_int_def(config_file, "DORIS_SERVER", "socket_recv_bufsize", &g_doris_server_info.sock_recv_bufsize, 524288); + pthread_mutex_init(&g_doris_server_info.mutex_lock, NULL); return 0; } static int doris_server_register_field_stat(struct doris_global_info *param) { const char *field_names[DRS_FSSTAT_FIELD_NUM]={"RecvErrVer", "FileStarts", "FileComplete", "ClientInvReq", - "ClientMetaReq", "SendNoNewMeta", "ClientFileReq", "SendBytes", "SendFile404"}; + "ClientMetaReq", "SendNoNewMeta", "ClientFileReq", "SendFiles", "SendBytes", "SendFile404", "VerExpire"}; const char *status_names[DRS_FSSTAT_STATUS_NUM]={"MemoryUsed"}; - const char *column_names[DRS_FSSTAT_CLUMN_NUM]={"RecvFullVer", "RecvIncVer", "RecvFiles", "SendResMeta", "SendFiles", + const char *column_names[DRS_FSSTAT_CLUMN_NUM]={"RecvFullVer", "RecvIncVer", "RecvFiles", "PostOnWay", "SendResMeta", "CurFullVer", "CurIncVer", "TotalCfgNum"}; int value; @@ -188,7 +272,8 @@ static int32_t doris_init_config_for_business(struct doris_global_info *info, st { char tmpbuffer[4096], tmp_dir[256], tmp_dir2[256], *bizname, *save=NULL; struct doris_business *business; - map::iterator iter; + map::iterator iter_csm; + map::iterator iter_prd; if(0>=MESA_load_profile_string_nodef(config_file, "DORIS_SERVER", "business_system_list", tmpbuffer, sizeof(tmpbuffer))) { @@ -214,7 +299,7 @@ static int32_t doris_init_config_for_business(struct doris_global_info *info, st } info->name2business->insert(make_pair(string(business->bizname), business)); - MESA_load_profile_uint_def(config_file, business->bizname, "grafana_monitor_status_id", &business->mm_status_codeid, 3); + MESA_load_profile_uint_def(config_file, business->bizname, "grafana_monitor_status_id", &business->mmval_status_codeid, 3); MESA_load_profile_uint_def(config_file, business->bizname, "mem_cache_max_versions", &business->cache_max_versions, 0); if(0>MESA_load_profile_string_nodef(config_file, business->bizname, "store_config_path", business->store_path_root, sizeof(business->store_path_root))) { @@ -230,28 +315,8 @@ static int32_t doris_init_config_for_business(struct doris_global_info *info, st } MESA_load_profile_uint_def(config_file, business->bizname, "receive_config_way", &business->recv_way, RECV_WAY_DRS_CLIENT); - if(business->recv_way == RECV_WAY_DRS_CLIENT) - { - if(0>=MESA_load_profile_string_nodef(config_file, business->bizname, "doris_client_confile", tmp_dir, sizeof(tmp_dir))) - { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]doris_client_confile not found!", config_file); - assert(0);return -1; - } - if((iter=info->confile2param->find(string(tmp_dir))) != info->confile2param->end()) - { - business->param = iter->second; - } - else - { - business->param = doris_parameter_new(tmp_dir, manage_evbase, info->log_runtime); - if(business->param == NULL) - { - assert(0);return -2; - } - info->confile2param->insert(make_pair(string(tmp_dir), business->param)); - } - } - else + assert(business->recv_way==RECV_WAY_IDX_FILE || business->recv_way==RECV_WAY_DRS_CLIENT || business->recv_way==RECV_WAY_HTTP_POST); + if(business->recv_way == RECV_WAY_IDX_FILE) { if(0>MESA_load_profile_string_nodef(config_file, business->bizname, "receive_config_path_full", business->recv_path_full, sizeof(business->recv_path_full))) { @@ -264,12 +329,52 @@ static int32_t doris_init_config_for_business(struct doris_global_info *info, st assert(0);return -1; } } + else //ͬҲҪ + { + if(business->recv_way==RECV_WAY_DRS_CLIENT || g_doris_server_info.cluster_sync_mode) + { + if(0>=MESA_load_profile_string_nodef(config_file, business->bizname, "doris_client_confile", tmp_dir, sizeof(tmp_dir))) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]doris_client_confile not found!", config_file); + assert(0);return -1; + } + if((iter_csm=info->confile2csmparam->find(string(tmp_dir))) != info->confile2csmparam->end()) + { + business->param_csum = iter_csm->second; + } + else + { + business->param_csum = doris_csum_parameter_new(tmp_dir, manage_evbase, info->log_runtime); + if(business->param_csum == NULL) + { + assert(0);return -2; + } + info->confile2csmparam->insert(make_pair(string(tmp_dir), business->param_csum)); + } + } + if(business->recv_way == RECV_WAY_HTTP_POST) //Ҫͬ + { + MESA_load_profile_uint_def(config_file, business->bizname, "producer_concurrence_allowed", &business->concurrency_allowed, 0); + if(MESA_load_profile_int_nodef(config_file, business->bizname, "producer_listen_port", &business->producer_port)<0) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [%s]producer_listen_port not found!", config_file, business->bizname); + assert(0);return -1; + } + if(g_doris_server_info.cluster_sync_mode && + NULL==(business->param_prod = doris_prod_parameter_new(tmp_dir, manage_evbase, info->log_runtime))) + { + assert(0);return -2; + } + g_doris_server_info.business_post_num++; + business->token2node = new map; + } + } business->fs_lineid = FS_register(info->fsstat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, business->bizname);; snprintf(tmp_dir, 512, "latest_cfg_version_%s", business->bizname); - business->mm_latest_ver = MESA_Monitor_register(info->monitor, tmp_dir, MONITOR_METRICS_GAUGE, "Latest doris config version."); + business->mmid_latest_ver = MESA_Monitor_register(info->monitor, tmp_dir, MONITOR_METRICS_GAUGE, "Latest doris config version."); snprintf(tmp_dir, 512, "total_config_num_%s", business->bizname); - business->mm_total_cfgnum = MESA_Monitor_register(info->monitor, tmp_dir, MONITOR_METRICS_GAUGE, "Total config num from latest full version till now."); + business->mmid_total_cfgnum = MESA_Monitor_register(info->monitor, tmp_dir, MONITOR_METRICS_GAUGE, "Total config num from latest full version till now."); } return 0; } @@ -302,14 +407,16 @@ int main(int argc, char **argv) struct timeval tv; struct event timer_statistic_fsstat; struct evhttp *manager_http; + char netname[32]; + memset(&g_doris_server_info, 0, sizeof(struct doris_global_info)); if(doris_read_profile_configs(NIRVANA_CONFIG_FILE) || doris_server_register_field_stat(&g_doris_server_info)) { return -1; } evthread_use_pthreads(); g_doris_server_info.name2business = new map; - g_doris_server_info.confile2param = new map; + g_doris_server_info.confile2csmparam = new map; manage_evbase = event_base_new(); /*Doris manager server*/ @@ -327,7 +434,7 @@ int main(int argc, char **argv) evhttp_set_cb(manager_http, "/doris/statistic/status", manager_statistic_status_requests_cb, NULL); evhttp_set_cb(manager_http, "/doris/statistic/threads", manager_statistic_threads_requests_cb, NULL); evhttp_set_gencb(manager_http, manager_generic_requests_cb, NULL); - g_doris_server_info.monitor = MESA_Monitor_instance_evhttp_new(manager_http, doris_vesion_20210804); + g_doris_server_info.monitor = MESA_Monitor_instance_evhttp_new(manager_http, doris_version_20210825); if(evhttp_accept_socket(manager_http, g_doris_server_info.manager)) { printf("evhttp_accept_socket %d error!\n", g_doris_server_info.manager); @@ -340,6 +447,24 @@ int main(int argc, char **argv) { return -4; } + if(g_doris_server_info.business_post_num > 0) + { + if((MESA_load_profile_string_nodef(NIRVANA_CONFIG_FILE, "DORIS_SERVER", "local_net_name", netname, sizeof(netname))<0) || + (g_doris_server_info.local_ip = get_ip_by_ifname(netname))<0) + { + printf("%s: [DORIS_SERVER]local_net_name not valid", NIRVANA_CONFIG_FILE); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]local_net_name not valid", NIRVANA_CONFIG_FILE); + assert(0);return -11; + } + g_doris_server_info.mmid_post_server = MESA_Monitor_register(g_doris_server_info.monitor, + "http_post_server_status", MONITOR_METRICS_GAUGE, "Running status of doris http post server."); + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mmid_post_server, MONITOR_VALUE_SET, PROMETHUES_POST_SERVER_DOWN); + } + + if(g_doris_server_info.ssl_conn_on && NULL==(g_doris_server_info.ssl_instance=doris_connections_create_ssl_ctx())) + { + assert(0);return -8; + } pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); @@ -353,6 +478,14 @@ int main(int argc, char **argv) assert(0);return -5; } } + else if(g_doris_server_info.business[i].recv_way == RECV_WAY_HTTP_POST) + { + if(pthread_create(&thread_desc, &attr, thread_http_post_recv_cfg, &g_doris_server_info.business[i])) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "pthread_create(): %s", strerror(errno)); + assert(0);return -6; + } + } else { if(pthread_create(&thread_desc, &attr, thread_index_file_recv_cfg, &g_doris_server_info.business[i])) @@ -364,17 +497,13 @@ int main(int argc, char **argv) } /*Doris http server*/ - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { - g_doris_server_info.listener = doris_create_listen_socket(g_doris_server_info.server_port); - if(g_doris_server_info.listener < 0) + g_doris_server_info.listener_csum = doris_create_listen_socket(g_doris_server_info.consumer_port); + if(g_doris_server_info.listener_csum < 0) { return -7; } - if(g_doris_server_info.ssl_conn_on && NULL==(g_doris_server_info.ssl_instance=doris_connections_create_ssl_ctx())) - { - assert(0);return -8; - } for(u_int32_t i=0; i @@ -39,35 +39,65 @@ using namespace std; #define RECV_WAY_IDX_FILE 2 #define RECV_WAY_HTTP_POST 3 +#define PROMETHUES_POST_SERVER_OK 0 +#define PROMETHUES_POST_SERVER_UPING 1 +#define PROMETHUES_POST_SERVER_DOWN 2 + struct doris_business { + /*first for configuration*/ char bizname[32]; u_int32_t recv_way; u_int32_t cache_max_versions; + u_int32_t concurrency_allowed; char recv_path_full[256]; char recv_path_inc[256]; char store_path_root[256]; struct version_list_handle *cfgver_head; - struct doris_parameter *param; - + struct doris_csum_param *param_csum; + struct doris_prod_param *param_prod; + u_int32_t ready_to_sync; + u_int32_t posts_on_the_way; + int32_t producer_port; //֤ÿbizserverֻ߳һ⽻instance + evutil_socket_t listener_prod; int64_t total_cfgnum; - int32_t mm_latest_ver; - int32_t mm_total_cfgnum; - u_int32_t mm_status_codeid; //MMڲ쳣״̬id + int32_t mmid_latest_ver; + int32_t mmid_total_cfgnum; + u_int32_t mmval_status_codeid; //MMڲ쳣״̬idGrafanavalue u_int32_t fs_lineid; pthread_rwlock_t rwlock; + + /*next for updating*/ + struct event_base *worker_evbase; + struct doris_prod_instance *instance; + map *token2node; + int64_t version; + int32_t source_from; + int32_t type; + int64_t version_cfgnum; + char inc_index_path[256]; //incĿ¼µȫ + char tmp_index_path[256]; //incĿ¼µȫ + char full_index_path[256]; //fullĿ¼µȫ + char cfg_file_path[256]; + FILE *fp_cfg_file; //ڶļDRS Clientӿ + FILE *fp_idx_file; //ڶļDRS Clientӿ + struct version_list_node *cur_vernode; }; struct doris_global_info { u_int32_t iothreads; - int32_t server_port; + int32_t consumer_port; int32_t manager_port; int32_t sock_recv_bufsize; u_int32_t ssl_conn_on; u_int32_t scan_idx_interval; u_int32_t cache_frag_size; - u_int32_t server_role_sw; + u_int32_t max_http_body_size; + u_int32_t idx_file_maat; + u_int32_t max_concurrent_reqs; + u_int32_t cluster_sync_mode; + u_int32_t post_vernode_ttl; char ssl_CA_path[256]; char ssl_cert_file[256]; @@ -76,16 +106,21 @@ struct doris_global_info pthread_mutex_t *lock_cs; SSL_CTX *ssl_instance; - evutil_socket_t listener; + evutil_socket_t listener_csum; evutil_socket_t manager; + u_int32_t token_seq; + int32_t local_ip; struct doris_business business[MAX_BUSINESS_NUM]; u_int32_t business_num; - u_int32_t idx_file_maat; + u_int32_t business_post_num; //postģʽм + int32_t business_post_ups; //˼ + int32_t mmid_post_server; //value=PROMETHUES_POST_* map *name2business; - map *confile2param; + map *confile2csmparam; struct MESA_MonitorHandler *monitor; + pthread_mutex_t mutex_lock; /*logs*/ u_int32_t log_level; @@ -106,6 +141,7 @@ struct doris_global_info }; int doris_mkdir_according_path(const char * path); +int doris_create_listen_socket(int bind_port); #endif diff --git a/server/doris_server_receive.cpp b/server/doris_server_receive.cpp index 1eecc55..6d75fe4 100644 --- a/server/doris_server_receive.cpp +++ b/server/doris_server_receive.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include "doris_server_main.h" #include "doris_server_scandir.h" #include "doris_server_receive.h" @@ -45,6 +47,8 @@ void config_table_node_cleanup(struct table_list_node *table_node) TAILQ_REMOVE(&table_node->frag_head, fragnode, frag_node); config_frag_node_cleanup(fragnode); } + config_frag_node_cleanup(table_node->cur_frag); + cJSON_Delete(table_node->table_meta); free(table_node); } @@ -59,10 +63,15 @@ void config_version_node_cleanup(struct version_list_node *vernode) TAILQ_REMOVE(&vernode->table_head, tablenode, table_node); config_table_node_cleanup(tablenode); } + config_table_node_cleanup(vernode->cur_table); free(vernode->metacont); cJSON_Delete(vernode->metajson); cJSON_Delete(vernode->arrayjson); cJSON_Delete(vernode->table_meta); + if(vernode->business!=NULL && vernode->business->recv_way==RECV_WAY_HTTP_POST) + { + vernode->business->token2node->erase(string(vernode->token)); + } free(vernode); } @@ -128,7 +137,7 @@ static void cfgver_delay_destroy_timer_cb(int fd, short kind, void *userp) free(delay_event); } -static void cfgver_handle_delay_destroy(struct confile_save *save, struct event_base *evbase, struct version_list_handle *version) +static void cfgver_handle_delay_destroy(struct event_base *evbase, struct version_list_handle *version) { struct common_timer_event *delay_event; @@ -139,186 +148,188 @@ static void cfgver_handle_delay_destroy(struct confile_save *save, struct event_ } /*fileϵкдļ*/ -void doris_config_file_version_start(struct doris_instance *instance, cJSON *meta, void *userdata) +void doris_config_file_version_start(struct doris_csum_instance *instance, cJSON *meta, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; - if(save->type == CFG_UPDATE_TYPE_FULL) + if(business->type == CFG_UPDATE_TYPE_FULL) { - snprintf(save->inc_index_path, 512, "%s/inc/index/full_config_index.%010lu", save->business->store_path_root, save->version); - snprintf(save->tmp_index_path, 512, "%s/inc/full_config_index.%010lu.ing", save->business->store_path_root, save->version); - snprintf(save->full_index_path, 512, "%s/full/index/full_config_index.%010lu", save->business->store_path_root, save->version); + snprintf(business->inc_index_path, 512, "%s/inc/index/full_config_index.%010lu", business->store_path_root, business->version); + snprintf(business->tmp_index_path, 512, "%s/inc/full_config_index.%010lu.ing", business->store_path_root, business->version); + snprintf(business->full_index_path, 512, "%s/full/index/full_config_index.%010lu", business->store_path_root, business->version); } else { - snprintf(save->inc_index_path, 512, "%s/inc/index/inc_config_index.%010lu", save->business->store_path_root, save->version); - snprintf(save->tmp_index_path, 512, "%s/inc/full_config_index.%010lu.ing", save->business->store_path_root, save->version); + snprintf(business->inc_index_path, 512, "%s/inc/index/inc_config_index.%010lu", business->store_path_root, business->version); + snprintf(business->tmp_index_path, 512, "%s/inc/full_config_index.%010lu.ing", business->store_path_root, business->version); } - if(NULL==(save->fp_idx_file = fopen(save->tmp_index_path, "w+"))) + if(NULL==(business->fp_idx_file = fopen(business->tmp_index_path, "w+"))) { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, fopen %s failed: %s", save->business->bizname, save->tmp_index_path, strerror(errno)); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, fopen %s failed: %s", business->bizname, business->tmp_index_path, strerror(errno)); assert(0); } } -void doris_config_file_version_finish(struct doris_instance *instance, void *userdata) +void doris_config_file_version_finish(struct doris_csum_instance *instance, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; - fclose(save->fp_idx_file); - if(rename(save->tmp_index_path, save->inc_index_path)) + fclose(business->fp_idx_file); + if(rename(business->tmp_index_path, business->inc_index_path)) { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, rename %s to %s failed: %s", save->business->bizname, save->tmp_index_path, save->inc_index_path, strerror(errno)); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, rename %s to %s failed: %s", + business->bizname, business->tmp_index_path, business->inc_index_path, strerror(errno)); assert(0); } - if(save->type == CFG_UPDATE_TYPE_FULL) + if(business->type == CFG_UPDATE_TYPE_FULL) { - if(link(save->inc_index_path, save->full_index_path) && errno!=EEXIST) //Ӳ + if(link(business->inc_index_path, business->full_index_path) && errno!=EEXIST) //Ӳ { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, rename %s to %s failed: %s", save->business->bizname, save->tmp_index_path, save->inc_index_path, strerror(errno)); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, rename %s to %s failed: %s", + business->bizname, business->tmp_index_path, business->inc_index_path, strerror(errno)); assert(0); } } - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, Version %lu write finished, index file: %s", save->business->bizname, save->version, save->inc_index_path); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, Version %lu write finished, index file: %s", + business->bizname, business->version, business->inc_index_path); } -void doris_config_file_version_error(struct doris_instance *instance, void *userdata) +void doris_config_file_version_error(struct doris_csum_instance *instance, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; - if(save->fp_idx_file != NULL) + if(business->fp_idx_file != NULL) { - fclose(save->fp_idx_file); - remove(save->tmp_index_path); + fclose(business->fp_idx_file); + remove(business->tmp_index_path); } - if(save->fp_cfg_file != NULL) + if(business->fp_cfg_file != NULL) { - fclose(save->fp_cfg_file); - remove(save->cfg_file_path); + fclose(business->fp_cfg_file); + remove(business->cfg_file_path); } - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, Version %llu error, rolling back...", save->business->bizname, save->version); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, Version %llu error, rolling back...", business->bizname, business->version); } -void doris_config_file_cfgfile_start(struct doris_instance *instance, +void doris_config_file_cfgfile_start(struct doris_csum_instance *instance, const struct tablemeta *meta, const char *localpath, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; struct tm *localtm, savetime; time_t now; const char *type; char dir[256]; - type = (save->type == CFG_UPDATE_TYPE_FULL)?"full":"inc"; + type = (business->type == CFG_UPDATE_TYPE_FULL)?"full":"inc"; now = time(NULL); localtm = localtime_r(&now, &savetime); - snprintf(dir, 256, "%s/%s/%04d-%02d-%02d", save->business->store_path_root, type, localtm->tm_year+1900, localtm->tm_mon+1, localtm->tm_mday); + snprintf(dir, 256, "%s/%s/%04d-%02d-%02d", business->store_path_root, type, localtm->tm_year+1900, localtm->tm_mon+1, localtm->tm_mday); if(access(dir, F_OK)) { doris_mkdir_according_path(dir); } - snprintf(save->cfg_file_path, 256, "%s/%s", dir, meta->filename); - if(g_doris_server_info.idx_file_maat) //MAATʽ֪ͨļ + snprintf(business->cfg_file_path, 256, "%s/%s", dir, meta->filename); + if(g_doris_server_info.idx_file_maat || meta->userregion==NULL) //MAATʽ֪ͨļ { - fprintf(save->fp_idx_file, "%s\t%u\t%s\n", meta->tablename, meta->cfgnum, save->cfg_file_path); + fprintf(business->fp_idx_file, "%s\t%u\t%s\n", meta->tablename, meta->cfgnum, business->cfg_file_path); } else //תɫûԶϢ { - fprintf(save->fp_idx_file, "%s\t%u\t%s\t%s\n", meta->tablename, meta->cfgnum, save->cfg_file_path, meta->userregion); + fprintf(business->fp_idx_file, "%s\t%u\t%s\t%s\n", meta->tablename, meta->cfgnum, business->cfg_file_path, meta->userregion); } - if(NULL == (save->fp_cfg_file = fopen(save->cfg_file_path, "w+"))) + if(NULL == (business->fp_cfg_file = fopen(business->cfg_file_path, "w+"))) { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, fopen %s failed: %s", save->business->bizname, save->cfg_file_path, strerror(errno)); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, fopen %s failed: %s", business->bizname, business->cfg_file_path, strerror(errno)); assert(0); } else { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, File %s start writing...", save->business->bizname, save->cfg_file_path); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, File %s start writing...", business->bizname, business->cfg_file_path); } } -void doris_config_file_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata) +void doris_config_file_cfgfile_update(struct doris_csum_instance *instance, const char *data, size_t len, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; size_t writen_len; - writen_len = fwrite(data, 1, len, save->fp_cfg_file); + writen_len = fwrite(data, 1, len, business->fp_cfg_file); if(writen_len != len) { - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, fwrite %s failed: %s", save->business->bizname, save->cfg_file_path, strerror(errno)); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, fwrite %s failed: %s", business->bizname, business->cfg_file_path, strerror(errno)); assert(0); } } -void doris_config_file_cfgfile_finish(struct doris_instance *instance, void *userdata) +void doris_config_file_cfgfile_finish(struct doris_csum_instance *instance, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; - fclose(save->fp_cfg_file); + struct doris_business *business=(struct doris_business *)userdata; + fclose(business->fp_cfg_file); - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, File %s write finished", save->business->bizname, save->cfg_file_path); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, File %s write finished", business->bizname, business->cfg_file_path); } /*memϵкڴ*/ -void doris_config_mem_version_start(struct doris_instance *instance, cJSON *meta, void *userdata) +void doris_config_mem_version_start(struct doris_csum_instance *instance, cJSON *meta, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; - save->cur_vernode = (struct version_list_node *)calloc(1, sizeof(struct version_list_node)); - TAILQ_INIT(&save->cur_vernode->table_head); - save->cur_vernode->metajson = cJSON_CreateObject(); - save->cur_vernode->arrayjson= cJSON_CreateArray(); + business->cur_vernode = (struct version_list_node *)calloc(1, sizeof(struct version_list_node)); + TAILQ_INIT(&business->cur_vernode->table_head); + business->cur_vernode->metajson = cJSON_CreateObject(); + business->cur_vernode->arrayjson= cJSON_CreateArray(); - save->cur_vernode->version = save->version; - cJSON_AddNumberToObject(save->cur_vernode->metajson, "version", save->cur_vernode->version); + business->cur_vernode->version = business->version; + cJSON_AddNumberToObject(business->cur_vernode->metajson, "version", business->cur_vernode->version); - save->cur_vernode->cfg_type = save->type; - cJSON_AddNumberToObject(save->cur_vernode->metajson, "type", save->cur_vernode->cfg_type); + business->cur_vernode->cfg_type = business->type; + cJSON_AddNumberToObject(business->cur_vernode->metajson, "type", business->cur_vernode->cfg_type); } -void doris_config_mem_version_finish(struct doris_instance *instance, void *userdata) +void doris_config_mem_version_finish(struct doris_csum_instance *instance, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; - struct version_list_handle *cur_version; + struct doris_business *business=(struct doris_business *)userdata; struct version_list_handle *tmplist; struct version_list_handle *cfgver_handle; - cJSON_AddItemToObject(save->cur_vernode->metajson, "configs", save->cur_vernode->arrayjson); - save->cur_vernode->arrayjson = NULL; - save->cur_vernode->metacont = cJSON_PrintUnformatted(save->cur_vernode->metajson); - save->cur_vernode->metalen = strlen(save->cur_vernode->metacont); - cJSON_Delete(save->cur_vernode->metajson); - save->cur_vernode->metajson = NULL; + cJSON_AddItemToObject(business->cur_vernode->metajson, "configs", business->cur_vernode->arrayjson); + business->cur_vernode->arrayjson = NULL; + business->cur_vernode->metacont = cJSON_PrintUnformatted(business->cur_vernode->metajson); + business->cur_vernode->metalen = strlen(business->cur_vernode->metacont); + cJSON_Delete(business->cur_vernode->metajson); + business->cur_vernode->metajson = NULL; - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, Version %lu mem finished, info: %s", save->business->bizname, save->version, save->cur_vernode->metacont); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, Version %lu mem finished, info: %s", business->bizname, business->version, business->cur_vernode->metacont); - if(save->cur_vernode->cfg_type==CFG_UPDATE_TYPE_FULL && save->business->cfgver_head->latest_version!=0) + if(business->cur_vernode->cfg_type==CFG_UPDATE_TYPE_FULL && business->cfgver_head->latest_version!=0) { - cur_version = config_version_handle_new(); - cur_version->latest_version = save->cur_vernode->version; - cur_version->version_num = 1; - TAILQ_INSERT_TAIL(&cur_version->version_head, save->cur_vernode, version_node); - cur_version->oldest_vernode = TAILQ_FIRST(&cur_version->version_head); - cur_version->version2node->insert(make_pair(cur_version->latest_version, save->cur_vernode)); + cfgver_handle = config_version_handle_new(); + cfgver_handle->latest_version = business->cur_vernode->version; + cfgver_handle->version_num = 1; + TAILQ_INSERT_TAIL(&cfgver_handle->version_head, business->cur_vernode, version_node); + cfgver_handle->oldest_vernode = TAILQ_FIRST(&cfgver_handle->version_head); + cfgver_handle->version2node->insert(make_pair(cfgver_handle->latest_version, business->cur_vernode)); - pthread_rwlock_wrlock(&save->business->rwlock); - tmplist = save->business->cfgver_head; - save->business->cfgver_head = cur_version; - pthread_rwlock_unlock(&save->business->rwlock); - cfgver_handle_delay_destroy(save, save->evbase, tmplist); + pthread_rwlock_wrlock(&business->rwlock); + tmplist = business->cfgver_head; + business->cfgver_head = cfgver_handle; + pthread_rwlock_unlock(&business->rwlock); + cfgver_handle_delay_destroy(business->worker_evbase, tmplist); } else { - pthread_rwlock_wrlock(&save->business->rwlock); - cfgver_handle = save->business->cfgver_head; - TAILQ_INSERT_TAIL(&cfgver_handle->version_head, save->cur_vernode, version_node); - cfgver_handle->latest_version = save->cur_vernode->version; - cfgver_handle->version2node->insert(make_pair(save->cur_vernode->version, save->cur_vernode)); + pthread_rwlock_wrlock(&business->rwlock); + cfgver_handle = business->cfgver_head; + TAILQ_INSERT_TAIL(&cfgver_handle->version_head, business->cur_vernode, version_node); + cfgver_handle->latest_version = business->cur_vernode->version; + cfgver_handle->version2node->insert(make_pair(business->cur_vernode->version, business->cur_vernode)); if(cfgver_handle->oldest_vernode == NULL) { cfgver_handle->oldest_vernode = TAILQ_FIRST(&cfgver_handle->version_head); } /*ļ໺N汾ԪϢȫ*/ - if(save->business->cache_max_versions!=0 && cfgver_handle->version_num>=save->business->cache_max_versions) + if(business->cache_max_versions!=0 && cfgver_handle->version_num>=business->cache_max_versions) { config_version_node_free_content(cfgver_handle->oldest_vernode); cfgver_handle->oldest_vernode = TAILQ_NEXT(cfgver_handle->oldest_vernode, version_node); @@ -327,281 +338,287 @@ void doris_config_mem_version_finish(struct doris_instance *instance, void *user { cfgver_handle->version_num += 1; } - pthread_rwlock_unlock(&save->business->rwlock); + pthread_rwlock_unlock(&business->rwlock); } - save->cur_vernode = NULL; + business->cur_vernode = NULL; } -void doris_config_mem_version_error(struct doris_instance *instance, void *userdata) +void doris_config_mem_version_error(struct doris_csum_instance *instance, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; - config_frag_node_cleanup(save->cur_frag); - config_table_node_cleanup(save->cur_table); - config_version_node_cleanup(save->cur_vernode); - save->cur_frag = NULL; - save->cur_table = NULL; - save->cur_vernode = NULL; + if(business->cur_vernode != NULL) + { + config_version_node_cleanup(business->cur_vernode); + } + business->cur_vernode = NULL; } -void doris_config_mem_cfgfile_start(struct doris_instance *instance, +void doris_config_mem_cfgfile_start(struct doris_csum_instance *instance, const struct tablemeta *meta, const char *localpath, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; + struct table_list_node *cur_table; - save->cur_vernode->table_meta = cJSON_CreateObject(); - cJSON_AddStringToObject(save->cur_vernode->table_meta, "tablename", meta->tablename); - cJSON_AddStringToObject(save->cur_vernode->table_meta, "filename", meta->filename); - cJSON_AddNumberToObject(save->cur_vernode->table_meta, "cfg_num", meta->cfgnum); - cJSON_AddNumberToObject(save->cur_vernode->table_meta, "size", meta->size); + business->cur_vernode->table_meta = cJSON_CreateObject(); + cJSON_AddStringToObject(business->cur_vernode->table_meta, "tablename", meta->tablename); + cJSON_AddStringToObject(business->cur_vernode->table_meta, "filename", meta->filename); + cJSON_AddNumberToObject(business->cur_vernode->table_meta, "cfg_num", meta->cfgnum); + cJSON_AddNumberToObject(business->cur_vernode->table_meta, "size", meta->size); if(meta->userregion != NULL) { - cJSON_AddStringToObject(save->cur_vernode->table_meta, "user_region", meta->userregion); + cJSON_AddStringToObject(business->cur_vernode->table_meta, "user_region", meta->userregion); } - save->cur_table = (struct table_list_node *)calloc(1, sizeof(struct table_list_node)); - save->cur_table->filesize = meta->size; - snprintf(save->cur_table->tablename, 64, "%s", meta->tablename); - snprintf(save->cur_table->localpath, 256, "%s", localpath); - TAILQ_INIT(&save->cur_table->frag_head); + cur_table = (struct table_list_node *)calloc(1, sizeof(struct table_list_node)); + cur_table->filesize = meta->size; + snprintf(cur_table->tablename, 64, "%s", meta->tablename); + snprintf(cur_table->localpath, 256, "%s", localpath); + TAILQ_INIT(&cur_table->frag_head); + business->cur_vernode->cur_table = cur_table; MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, table %s.%010llu start loading to memory...", - save->business->bizname, meta->tablename, save->version); + business->bizname, meta->tablename, business->version); } -void doris_config_mem_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata) +void doris_config_mem_cfgfile_update(struct doris_csum_instance *instance, const char *data, size_t len, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; + struct table_list_node *cur_table; size_t cache_len, offset=0; FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_status[DRS_FSSTAT_MEMORY_USED], 0, FS_OP_ADD, len); + + cur_table = business->cur_vernode->cur_table; while(len > 0) { - if(save->cur_frag == NULL) + if(cur_table->cur_frag == NULL) { - save->cur_frag = (struct cont_frag_node *)calloc(1, sizeof(struct cont_frag_node)); - save->cur_frag->start = save->cur_table->cur_totallen; - save->cur_frag->totalsize = save->cur_table->filesize - save->cur_table->cur_totallen; - if(save->cur_frag->totalsize > g_doris_server_info.cache_frag_size) + cur_table->cur_frag = (struct cont_frag_node *)calloc(1, sizeof(struct cont_frag_node)); + cur_table->cur_frag->start = cur_table->cur_totallen; + cur_table->cur_frag->totalsize = cur_table->filesize - cur_table->cur_totallen; + if(cur_table->filesize==0 || cur_table->cur_frag->totalsize > g_doris_server_info.cache_frag_size) { - save->cur_frag->totalsize = g_doris_server_info.cache_frag_size; + cur_table->cur_frag->totalsize = g_doris_server_info.cache_frag_size; } - save->cur_frag->end = save->cur_frag->start + save->cur_frag->totalsize - 1; - save->cur_frag->content = (char *)malloc(save->cur_frag->totalsize); + cur_table->cur_frag->end = cur_table->cur_frag->start + cur_table->cur_frag->totalsize - 1; + cur_table->cur_frag->content = (char *)malloc(cur_table->cur_frag->totalsize); } - if(save->cur_frag->totalsize > save->cur_frag->cur_fraglen + len) + if(cur_table->cur_frag->totalsize > cur_table->cur_frag->cur_fraglen + len) { - memcpy(save->cur_frag->content+save->cur_frag->cur_fraglen, data+offset, len); - save->cur_frag->cur_fraglen += len; - save->cur_table->cur_totallen += len; + memcpy(cur_table->cur_frag->content+cur_table->cur_frag->cur_fraglen, data+offset, len); + cur_table->cur_frag->cur_fraglen += len; + cur_table->cur_totallen += len; offset += len; len = 0; } else { - cache_len = save->cur_frag->totalsize - save->cur_frag->cur_fraglen; - memcpy(save->cur_frag->content+save->cur_frag->cur_fraglen, data+offset, cache_len); - save->cur_frag->cur_fraglen += cache_len; - save->cur_table->cur_totallen += cache_len; + cache_len = cur_table->cur_frag->totalsize - cur_table->cur_frag->cur_fraglen; + memcpy(cur_table->cur_frag->content+cur_table->cur_frag->cur_fraglen, data+offset, cache_len); + cur_table->cur_frag->cur_fraglen += cache_len; + cur_table->cur_totallen += cache_len; offset += cache_len; len -= cache_len; - TAILQ_INSERT_TAIL(&save->cur_table->frag_head, save->cur_frag, frag_node); - assert(save->cur_frag->cur_fraglen == save->cur_frag->end - save->cur_frag->start + 1); - save->cur_frag = NULL; + TAILQ_INSERT_TAIL(&cur_table->frag_head, cur_table->cur_frag, frag_node); + assert(cur_table->cur_frag->cur_fraglen == cur_table->cur_frag->end - cur_table->cur_frag->start + 1); + cur_table->cur_frag = NULL; } } - assert(save->cur_table->cur_totallen <= save->cur_table->filesize); + assert(cur_table->cur_totallen <= cur_table->filesize || cur_table->filesize==0); } -void doris_config_mem_cfgfile_finish(struct doris_instance *instance, const char *md5, void *userdata) +void doris_config_mem_cfgfile_finish(struct doris_csum_instance *instance, const char *md5, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; + struct table_list_node *cur_table; - cJSON_AddStringToObject(save->cur_vernode->table_meta, "md5", md5); - cJSON_AddItemToArray(save->cur_vernode->arrayjson, save->cur_vernode->table_meta); - save->cur_vernode->table_meta = NULL; + cJSON_AddStringToObject(business->cur_vernode->table_meta, "md5", md5); + cJSON_AddItemToArray(business->cur_vernode->arrayjson, business->cur_vernode->table_meta); + business->cur_vernode->table_meta = NULL; - if(save->cur_frag != NULL) + cur_table = business->cur_vernode->cur_table; + if(cur_table->cur_frag != NULL) { - TAILQ_INSERT_TAIL(&save->cur_table->frag_head, save->cur_frag, frag_node); - assert(save->cur_frag->cur_fraglen == save->cur_frag->end - save->cur_frag->start + 1); - save->cur_frag = NULL; + TAILQ_INSERT_TAIL(&cur_table->frag_head, cur_table->cur_frag, frag_node); + assert(cur_table->cur_frag->cur_fraglen == cur_table->cur_frag->end - cur_table->cur_frag->start + 1); + cur_table->cur_frag = NULL; } - assert(save->cur_table->cur_totallen == save->cur_table->filesize); - TAILQ_INSERT_TAIL(&save->cur_vernode->table_head, save->cur_table, table_node); - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, table %s.%010llu load to memory finished", save->business->bizname, save->cur_table->tablename, save->version); - save->cur_table = NULL; + assert(cur_table->cur_totallen == cur_table->filesize); + TAILQ_INSERT_TAIL(&business->cur_vernode->table_head, cur_table, table_node); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, table %s.%010llu load to memory finished", business->bizname, cur_table->tablename, business->version); + business->cur_vernode->cur_table = NULL; } /*commonϵкʱ*/ -void doris_config_common_version_start(struct confile_save *save, cJSON *meta) +void doris_config_common_version_start(struct doris_business *business, cJSON *meta) { cJSON *sub; sub = cJSON_GetObjectItem(meta, "version"); - save->version = sub->valuedouble; + business->version = sub->valuedouble; sub = cJSON_GetObjectItem(meta, "type"); - save->type = sub->valueint; - assert(save->type==CFG_UPDATE_TYPE_FULL || save->type==CFG_UPDATE_TYPE_INC); - save->version_cfgnum = 0; - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, Version %lu start updating...", save->business->bizname, save->version); + business->type = sub->valueint; + assert(business->type==CFG_UPDATE_TYPE_FULL || business->type==CFG_UPDATE_TYPE_INC); + business->version_cfgnum = 0; + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, Version %lu start updating...", business->bizname, business->version); } -void doris_config_common_version_finish(struct confile_save *save) +void doris_config_common_version_finish(struct doris_business *business) { - if(save->type == CFG_UPDATE_TYPE_FULL) + if(business->type == CFG_UPDATE_TYPE_FULL) { - save->business->total_cfgnum = save->version_cfgnum; - FS_operate(g_doris_server_info.fsstat_handle, save->business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_CUR_FULL_VERSION], FS_OP_SET, save->version); - FS_operate(g_doris_server_info.fsstat_handle, save->business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_CONFIG_TOTAL_NUM], FS_OP_SET, save->version_cfgnum); - FS_operate(g_doris_server_info.fsstat_handle, save->business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_RECV_FULL_VER], FS_OP_ADD, 1); + business->total_cfgnum = business->version_cfgnum; + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_CUR_FULL_VERSION], FS_OP_SET, business->version); + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_CONFIG_TOTAL_NUM], FS_OP_SET, business->version_cfgnum); + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_RECV_FULL_VER], FS_OP_ADD, 1); } else { - save->business->total_cfgnum += save->version_cfgnum; - FS_operate(g_doris_server_info.fsstat_handle, save->business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_CUR_INC_VERSION], FS_OP_SET, save->version); - FS_operate(g_doris_server_info.fsstat_handle, save->business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_CONFIG_TOTAL_NUM], FS_OP_ADD, save->version_cfgnum); - FS_operate(g_doris_server_info.fsstat_handle, save->business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_RECV_INC_VER], FS_OP_ADD, 1); + business->total_cfgnum += business->version_cfgnum; + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_CUR_INC_VERSION], FS_OP_SET, business->version); + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_CONFIG_TOTAL_NUM], FS_OP_ADD, business->version_cfgnum); + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_RECV_INC_VER], FS_OP_ADD, 1); } - MESA_Monitor_operation(g_doris_server_info.monitor, save->business->mm_latest_ver, MONITOR_VALUE_SET, save->version); - MESA_Monitor_operation(g_doris_server_info.monitor, save->business->mm_total_cfgnum, MONITOR_VALUE_SET, save->business->total_cfgnum); - MESA_Monitor_set_status_code(g_doris_server_info.monitor, MONITOR_STATUS_OP_CLEAR, save->business->mm_status_codeid, NULL, NULL); - MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, Version %lu update finished", save->business->bizname, save->version); - } + MESA_Monitor_operation(g_doris_server_info.monitor, business->mmid_latest_ver, MONITOR_VALUE_SET, business->version); + MESA_Monitor_operation(g_doris_server_info.monitor, business->mmid_total_cfgnum, MONITOR_VALUE_SET, business->total_cfgnum); + MESA_Monitor_set_status_code(g_doris_server_info.monitor, MONITOR_STATUS_OP_CLEAR, business->mmval_status_codeid, NULL, NULL); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, Version %lu update finished", business->bizname, business->version); +} -void doris_config_common_version_error(struct confile_save *save) +void doris_config_common_version_error(struct doris_business *business) { FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_RECV_ERR_VER], 0, FS_OP_ADD, 1); //Grafana+Promethuesչʾڲ쳣״̬ - MESA_Monitor_set_status_code(g_doris_server_info.monitor, MONITOR_STATUS_OP_SET, save->business->mm_status_codeid, + MESA_Monitor_set_status_code(g_doris_server_info.monitor, MONITOR_STATUS_OP_SET, business->mmval_status_codeid, "Version receive error", "Receive config file error, please check producer"); } -void doris_config_common_cfgfile_start(struct confile_save *save, u_int32_t cfgnum) +void doris_config_common_cfgfile_start(struct doris_business *business, u_int32_t cfgnum) { - save->version_cfgnum += cfgnum; + business->version_cfgnum += cfgnum; FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_RECV_START_FILES], 0, FS_OP_ADD, 1); } -void doris_config_common_cfgfile_finish(struct confile_save *save) +void doris_config_common_cfgfile_finish(struct doris_business *business) { FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_RECV_CMPLT_FILES], 0, FS_OP_ADD, 1); - FS_operate(g_doris_server_info.fsstat_handle, save->business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_RECV_FILES], FS_OP_ADD, 1); + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_RECV_FILES], FS_OP_ADD, 1); } /*localmemϵкʱӱػĻص*/ -void doris_config_localmem_version_start(struct doris_instance *instance, cJSON *meta, void *userdata) +void doris_config_localmem_version_start(struct doris_csum_instance *instance, cJSON *meta, void *userdata) { - doris_config_common_version_start((struct confile_save *)userdata, meta); - if(g_doris_server_info.server_role_sw) + doris_config_common_version_start((struct doris_business *)userdata, meta); + if(g_doris_server_info.consumer_port) { doris_config_mem_version_start(instance, meta, userdata); } } -void doris_config_localmem_version_finish(struct doris_instance *instance, void *userdata) +void doris_config_localmem_version_finish(struct doris_csum_instance *instance, void *userdata) { - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { doris_config_mem_version_finish(instance, userdata); } - doris_config_common_version_finish((struct confile_save *)userdata); + doris_config_common_version_finish((struct doris_business *)userdata); } -void doris_config_localmem_version_error(struct doris_instance *instance, void *userdata) +void doris_config_localmem_version_error(struct doris_csum_instance *instance, void *userdata) { - doris_config_common_version_error((struct confile_save *)userdata); - if(g_doris_server_info.server_role_sw) + doris_config_common_version_error((struct doris_business *)userdata); + if(g_doris_server_info.consumer_port) { doris_config_mem_version_error(instance, userdata); } } -void doris_config_localmem_cfgfile_start(struct doris_instance *instance, +void doris_config_localmem_cfgfile_start(struct doris_csum_instance *instance, const struct tablemeta *meta, const char *localpath, void *userdata) { - doris_config_common_cfgfile_start((struct confile_save *)userdata, meta->cfgnum); - if(g_doris_server_info.server_role_sw) + doris_config_common_cfgfile_start((struct doris_business *)userdata, meta->cfgnum); + if(g_doris_server_info.consumer_port) { doris_config_mem_cfgfile_start(instance, meta, localpath, userdata); } } -void doris_config_localmem_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata) +void doris_config_localmem_cfgfile_update(struct doris_csum_instance *instance, const char *data, size_t len, void *userdata) { - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { doris_config_mem_cfgfile_update(instance, data, len, userdata); } } -void doris_config_localmem_cfgfile_finish(struct doris_instance *instance, const char *md5, void *userdata) +void doris_config_localmem_cfgfile_finish(struct doris_csum_instance *instance, const char *md5, void *userdata) { - doris_config_common_cfgfile_finish((struct confile_save *)userdata); - if(g_doris_server_info.server_role_sw) + doris_config_common_cfgfile_finish((struct doris_business *)userdata); + if(g_doris_server_info.consumer_port) { doris_config_mem_cfgfile_finish(instance, md5, userdata); } } /*ޱϵкʱص*/ -void doris_config_version_start(struct doris_instance *instance, cJSON *meta, void *userdata) +void doris_config_version_start(struct doris_csum_instance *instance, cJSON *meta, void *userdata) { - doris_config_common_version_start((struct confile_save *)userdata, meta); + doris_config_common_version_start((struct doris_business *)userdata, meta); doris_config_file_version_start(instance, meta, userdata); - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { doris_config_mem_version_start(instance, meta, userdata); } } -void doris_config_version_finish(struct doris_instance *instance, void *userdata) +void doris_config_version_finish(struct doris_csum_instance *instance, void *userdata) { doris_config_file_version_finish(instance, userdata); - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { doris_config_mem_version_finish(instance, userdata); } - doris_config_common_version_finish((struct confile_save *)userdata); + doris_config_common_version_finish((struct doris_business *)userdata); } -void doris_config_version_error(struct doris_instance *instance, void *userdata) +void doris_config_version_error(struct doris_csum_instance *instance, void *userdata) { - doris_config_common_version_error((struct confile_save *)userdata); + doris_config_common_version_error((struct doris_business *)userdata); doris_config_file_version_error(instance, userdata); - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { doris_config_mem_version_error(instance, userdata); } } -void doris_config_cfgfile_start(struct doris_instance *instance, +void doris_config_cfgfile_start(struct doris_csum_instance *instance, const struct tablemeta *meta, const char *localpath, void *userdata) { - struct confile_save *save=(struct confile_save *)userdata; + struct doris_business *business=(struct doris_business *)userdata; - doris_config_common_cfgfile_start((struct confile_save *)userdata, meta->cfgnum); + doris_config_common_cfgfile_start((struct doris_business *)userdata, meta->cfgnum); doris_config_file_cfgfile_start(instance, meta, localpath, userdata); - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { - doris_config_mem_cfgfile_start(instance, meta, save->cfg_file_path, userdata); + doris_config_mem_cfgfile_start(instance, meta, business->cfg_file_path, userdata); } } -void doris_config_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata) +void doris_config_cfgfile_update(struct doris_csum_instance *instance, const char *data, size_t len, void *userdata) { doris_config_file_cfgfile_update(instance, data, len, userdata); - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { doris_config_mem_cfgfile_update(instance, data, len, userdata); } } -void doris_config_cfgfile_finish(struct doris_instance *instance, const char *md5, void *userdata) +void doris_config_cfgfile_finish(struct doris_csum_instance *instance, const char *md5, void *userdata) { - doris_config_common_cfgfile_finish((struct confile_save *)userdata); + doris_config_common_cfgfile_finish((struct doris_business *)userdata); doris_config_file_cfgfile_finish(instance, userdata); - if(g_doris_server_info.server_role_sw) + if(g_doris_server_info.consumer_port) { doris_config_mem_cfgfile_finish(instance, md5, userdata); } @@ -611,22 +628,19 @@ void* thread_doris_client_recv_cfg(void *arg) { struct doris_business *business=(struct doris_business *)arg; struct event_base *client_evbase; - struct doris_instance *instance; + struct doris_csum_instance *instance; struct doris_callbacks doris_cbs; struct doris_arguments doris_args; struct doris_idxfile_scanner *scanner; enum DORIS_UPDATE_TYPE update_type; - struct confile_save save; char stored_path[512]; prctl(PR_SET_NAME, "client_recv"); client_evbase = event_base_new(); - memset(&save, 0, sizeof(struct confile_save)); - save.source_from = RECV_WAY_IDX_FILE; - save.evbase = client_evbase; - save.business = business; + business->source_from = RECV_WAY_IDX_FILE; + business->worker_evbase = client_evbase; scanner = doris_index_file_scanner(0); @@ -637,15 +651,14 @@ void* thread_doris_client_recv_cfg(void *arg) doris_cbs.cfgfile_start = doris_config_localmem_cfgfile_start; doris_cbs.cfgfile_update = doris_config_localmem_cfgfile_update; doris_cbs.cfgfile_finish = doris_config_localmem_cfgfile_finish; - doris_cbs.userdata = &save; + doris_cbs.version_updated= NULL; + doris_cbs.userdata = business; snprintf(stored_path, 512, "%s/full/index", business->store_path_root); update_type = doris_index_file_traverse(scanner, stored_path, &doris_cbs, NULL, g_doris_server_info.log_runtime); - assert(update_type!=CFG_UPDATE_TYPE_ERR); snprintf(stored_path, 512, "%s/inc/index", business->store_path_root); do { update_type = doris_index_file_traverse(scanner, stored_path, &doris_cbs, NULL, g_doris_server_info.log_runtime); - assert(update_type!=CFG_UPDATE_TYPE_ERR); }while(update_type != CFG_UPDATE_TYPE_NONE); @@ -657,11 +670,11 @@ void* thread_doris_client_recv_cfg(void *arg) doris_cbs.cfgfile_update = doris_config_cfgfile_update; doris_cbs.cfgfile_finish = doris_config_cfgfile_finish; - save.source_from = RECV_WAY_DRS_CLIENT; + business->source_from = RECV_WAY_DRS_CLIENT; memset(&doris_args, 0, sizeof(struct doris_arguments)); doris_args.current_version = scanner->cur_version; sprintf(doris_args.bizname, "%s", business->bizname); - instance = doris_instance_new(business->param, client_evbase, &doris_cbs, &doris_args, g_doris_server_info.log_runtime); + instance = doris_csum_instance_new(business->param_csum, client_evbase, &doris_cbs, &doris_args, g_doris_server_info.log_runtime); if(instance == NULL) { assert(0);return NULL; @@ -693,7 +706,6 @@ void* thread_index_file_recv_cfg(void *arg) { struct doris_business *business=(struct doris_business *)arg; struct event_base *client_evbase; - struct confile_save save; struct timeval tv; struct scanner_timer_priv timer_priv; enum DORIS_UPDATE_TYPE update_type; @@ -701,14 +713,12 @@ void* thread_index_file_recv_cfg(void *arg) prctl(PR_SET_NAME, "index_file"); - memset(&save, 0, sizeof(struct confile_save)); memset(&timer_priv, 0, sizeof(struct scanner_timer_priv)); client_evbase = event_base_new(); - save.source_from = RECV_WAY_IDX_FILE; - save.evbase = client_evbase; - save.business = business; + business->source_from = RECV_WAY_IDX_FILE; + business->worker_evbase = client_evbase; timer_priv.scanner = doris_index_file_scanner(0); timer_priv.business = business; @@ -720,16 +730,15 @@ void* thread_index_file_recv_cfg(void *arg) timer_priv.doris_cbs.cfgfile_start = doris_config_localmem_cfgfile_start; timer_priv.doris_cbs.cfgfile_update = doris_config_localmem_cfgfile_update; timer_priv.doris_cbs.cfgfile_finish = doris_config_localmem_cfgfile_finish; - timer_priv.doris_cbs.userdata = &save; + timer_priv.doris_cbs.version_updated= NULL; + timer_priv.doris_cbs.userdata = business; snprintf(stored_path, 512, "%s/full/index", business->store_path_root); update_type = doris_index_file_traverse(timer_priv.scanner, stored_path, &timer_priv.doris_cbs, NULL, g_doris_server_info.log_runtime); - assert(update_type!=CFG_UPDATE_TYPE_ERR); snprintf(stored_path, 512, "%s/inc/index", business->store_path_root); do{ update_type = doris_index_file_traverse(timer_priv.scanner, stored_path, &timer_priv.doris_cbs, NULL, g_doris_server_info.log_runtime); - assert(update_type!=CFG_UPDATE_TYPE_ERR); - }while(update_type!=CFG_UPDATE_TYPE_NONE && update_type!=CFG_UPDATE_TYPE_ERR); + }while(update_type!=CFG_UPDATE_TYPE_NONE); /*Check new configs*/ @@ -742,8 +751,7 @@ void* thread_index_file_recv_cfg(void *arg) update_type = doris_index_file_traverse(timer_priv.scanner, business->recv_path_full, &timer_priv.doris_cbs, NULL, g_doris_server_info.log_runtime); - assert(update_type!=CFG_UPDATE_TYPE_ERR); - if(update_type!=CFG_UPDATE_TYPE_NONE && update_type!=CFG_UPDATE_TYPE_ERR) + if(update_type!=CFG_UPDATE_TYPE_NONE) { tv.tv_sec = 0; } @@ -761,3 +769,1376 @@ void* thread_index_file_recv_cfg(void *arg) return NULL; } +struct bufferevent *doris_https_bufferevent_cb(struct event_base *evabse, void *arg) +{ + SSL_CTX *ssl_instance = (SSL_CTX *)arg; + + return bufferevent_openssl_socket_new(evabse, -1, SSL_new(ssl_instance), BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_CLOSE_ON_FREE); +} + +struct doris_business *lookup_bizstruct_from_name(const struct evkeyvalq *params) +{ + map::iterator iter; + const char *bizname; + + if(NULL == (bizname = evhttp_find_header(params, "business"))) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + return NULL; + } + if((iter = g_doris_server_info.name2business->find(string(bizname)))==g_doris_server_info.name2business->end()) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + return NULL; + } + return iter->second; +} + +struct version_list_node *lookup_vernode_struct_from_name(struct doris_business *business, const struct evkeyvalq *params) +{ + map::iterator iter; + const char *token; + + if(NULL == (token = evhttp_find_header(params, "token"))) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + return NULL; + } + if((iter = business->token2node->find(string(token)))==business->token2node->end()) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + return NULL; + } + return iter->second; +} + +struct version_list_node *lookup_vernode_struct_from_name_renew(struct doris_business *business, const struct evkeyvalq *params) +{ + struct version_list_node *vernode; + struct timeval tv; + + if(NULL == (vernode = lookup_vernode_struct_from_name(business, params))) + { + return NULL; + } + if(vernode->business->concurrency_allowed) + { + tv.tv_sec = g_doris_server_info.post_vernode_ttl; + tv.tv_usec = 0; + evtimer_add(&vernode->timer_expire, &tv); + } + return vernode; +} + +/*֤business֮ɵtokenͻ*/ +void prod_server_generate_token(struct doris_business *business, char *token/*OUT*/, size_t size) +{ + pthread_mutex_lock(&g_doris_server_info.mutex_lock); + snprintf(token, size, "%u-%lu-%u-%u", g_doris_server_info.local_ip, time(NULL), rand(), ++g_doris_server_info.token_seq); + pthread_mutex_unlock(&g_doris_server_info.mutex_lock); +} + +void business_resume_sync_peer_normal(struct doris_business *business) +{ + u_int32_t business_post_ups; + + if(!g_doris_server_info.cluster_sync_mode) + { + return; + } + + if(1 == atomic_set(&business->ready_to_sync, 1) || business->listener_prod==0) + { + return; + } + + pthread_mutex_lock(&g_doris_server_info.mutex_lock); + business_post_ups = ++g_doris_server_info.business_post_ups; + pthread_mutex_unlock(&g_doris_server_info.mutex_lock); + if(business_post_ups == g_doris_server_info.business_post_num) + { + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mmid_post_server, MONITOR_VALUE_SET, PROMETHUES_POST_SERVER_OK); + } + else + { + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mmid_post_server, MONITOR_VALUE_SET, PROMETHUES_POST_SERVER_UPING); + } + assert(business_post_ups <= g_doris_server_info.business_post_num); +} + +void business_set_sync_peer_abnormal(struct doris_business *business) +{ + u_int32_t business_post_ups; + + if(!g_doris_server_info.cluster_sync_mode) + { + return; + } + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "\033[1;31;40mcluster sync error, please check slave status!!!\033[0m\n"); + + if(0 == atomic_set(&business->ready_to_sync, 0) || business->listener_prod==0) + { + return; + } + pthread_mutex_lock(&g_doris_server_info.mutex_lock); + business_post_ups = --g_doris_server_info.business_post_ups; + pthread_mutex_unlock(&g_doris_server_info.mutex_lock); + if(business_post_ups == 0) + { + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mmid_post_server, MONITOR_VALUE_SET, PROMETHUES_POST_SERVER_DOWN); + } + else + { + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mmid_post_server, MONITOR_VALUE_SET, PROMETHUES_POST_SERVER_UPING); + } + assert(business_post_ups < g_doris_server_info.business_post_num); +} + +char *vernode_print_json_meta(struct version_list_node *vernode) +{ + struct table_list_node *tablenode; + cJSON *root, *array=NULL, *item; + char *p; + + root = cJSON_CreateObject(); + cJSON_AddStringToObject(root, "token", vernode->token); + cJSON_AddNumberToObject(root, "type", vernode->cfg_type); + TAILQ_FOREACH(tablenode, &vernode->table_head, table_node) + { + if(array == NULL) + { + array = cJSON_CreateArray(); + } + item = cJSON_CreateObject(); + cJSON_AddStringToObject(item, "tablename", tablenode->tablename); + cJSON_AddNumberToObject(item, "size", tablenode->cur_totallen); + cJSON_AddItemToArray(array, item); + assert(tablenode->finished); //ϴϵIJܼ + } + if(vernode->cur_table != NULL) + { + if(array == NULL) + { + array = cJSON_CreateArray(); + } + item = cJSON_CreateObject(); + cJSON_AddStringToObject(item, "tablename", vernode->cur_table->tablename); + cJSON_AddNumberToObject(item, "offset", vernode->cur_table->cur_totallen); + cJSON_AddItemToArray(array, item); + } + cJSON_AddItemToObject(root, "configs", array); + p = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + return p; +} + +void http_prod_server_verion_check_cb(struct evhttp_request *req, void *arg) +{ + struct doris_business *business=(struct doris_business *)arg; + struct version_list_node *vernode; + struct evkeyvalq params; + struct evbuffer *evbuf; + char *p; + + if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid"); + return; + } + if(NULL == (vernode = lookup_vernode_struct_from_name(business, ¶ms))) + { + evhttp_clear_headers(¶ms); + evhttp_send_error(req, HTTP_NOTFOUND, "Parameter token not found"); + return; + } + evhttp_clear_headers(¶ms); + + if(vernode->syncing) + { + evhttp_send_error(req, 310, "table syncing now, retry later"); + return; + } + + p = vernode_print_json_meta(vernode); + + evbuf = evbuffer_new(); + evbuffer_add(evbuf, p, strlen(p)); + if(vernode->version_finished) + { + evhttp_send_reply(req, HTTP_OK, "OK", evbuf); + } + else + { + evhttp_send_reply(req, 300, "version is posting", evbuf); + } + evbuffer_free(evbuf); + free(p); +} + +void http_config_direct_version_cancel(struct version_list_node *vernode, struct evhttp_request *req) +{ + struct doris_business *business=vernode->business; + struct table_list_node *tablenode; + char token[64]; + + sprintf(token, "%s", vernode->token); + if(vernode->synctx != NULL) + { + doris_prod_upload_ctx_destroy(vernode->synctx); + } + if(vernode->fp_idx_file != NULL) + { + fclose(vernode->fp_idx_file); + remove(vernode->tmp_index_path); + } + if(vernode->cur_table!=NULL && vernode->cur_table->fp_cfg_file != NULL) + { + fclose(vernode->cur_table->fp_cfg_file); + remove(vernode->cur_table->localpath); + } + TAILQ_FOREACH(tablenode, &vernode->table_head, table_node) + { + remove(tablenode->localpath); + } + config_version_node_cleanup(vernode); + if(evtimer_pending(&vernode->timer_expire, NULL)) + { + evtimer_del(&vernode->timer_expire); + } + business->cur_vernode = NULL; // + business->posts_on_the_way--; + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_POST_ON_THE_WAY], FS_OP_SET, business->posts_on_the_way); + + evhttp_send_reply(req, 200, "OK", NULL); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, post server version cancel, token: %s", business->bizname, token); +} + +void prod_sync_vercancel_result_cb(enum PROD_VEROP_RES result, void *userdata) +{ + struct version_list_node *vernode=(struct version_list_node *)userdata; + + vernode->syncing = 0; + vernode->retry_times++; + switch(result) + { + case VERSIONOP_RES_OK: + http_config_direct_version_cancel(vernode, vernode->req); + break; + + case VERSIONOP_RES_ERROR: + evhttp_send_error(vernode->req, 500, "version cancel sync error res_code"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "\033[1;31;40mbusiness: %s, version cancel sync error res_code, abandon it. Send 500 response to client.\033[0m", vernode->business->bizname); + break; + + case VERSIONOP_CURL_ERROR: + if(atomic_read(&vernode->business->ready_to_sync) && (vernode->retry_times < 3)) + { + vernode->syncing = 1; + doris_prod_version_cancel(vernode->synctx, prod_sync_vercancel_result_cb, vernode); + } + else + { + http_config_direct_version_cancel(vernode, vernode->req); + business_set_sync_peer_abnormal(vernode->business); + } + break; + default: assert(0);break; + } +} + +void http_prod_server_verion_cancel_cb(struct evhttp_request *req, void *arg) +{ + struct doris_business *business=(struct doris_business *)arg; + struct version_list_node *vernode; + struct evkeyvalq params; + + if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid"); + return; + } + if(NULL == (vernode = lookup_vernode_struct_from_name_renew(business, ¶ms))) + { + evhttp_clear_headers(¶ms); + evhttp_send_error(req, HTTP_OK, "Parameter token not found"); //һ + return; + } + evhttp_clear_headers(¶ms); + + if(vernode->version_finished) + { + evhttp_send_error(req, HTTP_BADREQUEST, "version already finished"); + return; + } + if(vernode->syncing) + { + evhttp_send_error(req, 300, "table syncing now, retry later"); + return; + } + + if(!atomic_read(&business->ready_to_sync) || + NULL!=evhttp_find_header(evhttp_request_get_input_headers(req), "X-Doris-Master-Slave-Sync")) + { + return http_config_direct_version_cancel(vernode, req); + } + vernode->retry_times = 0; + vernode->syncing = 1; + vernode->req = req; + doris_prod_version_cancel(vernode->synctx, prod_sync_vercancel_result_cb, vernode); +} + +void doris_config_post_version_finish(struct doris_business *business, struct version_list_node *vernode, int64_t newversion) +{ + assert(newversion > vernode->version); + vernode->version = newversion; + + if(vernode->cfg_type == CFG_UPDATE_TYPE_FULL) + { + snprintf(business->inc_index_path, 512, "%s/inc/index/full_config_index.%010lu", business->store_path_root, vernode->version); + snprintf(business->full_index_path, 512, "%s/full/index/full_config_index.%010lu", business->store_path_root, vernode->version); + } + else + { + snprintf(business->inc_index_path, 512, "%s/inc/index/inc_config_index.%010lu", business->store_path_root, vernode->version); + } + /*HTTP postʱ汾ÿԼʱ֪ͨļñļĹرպ*/ + sprintf(business->tmp_index_path, "%s", vernode->tmp_index_path); + business->version = vernode->version; + business->type = vernode->cfg_type; + business->fp_idx_file = vernode->fp_idx_file; + doris_config_file_version_finish(NULL, business); + vernode->fp_idx_file = NULL; + + if(g_doris_server_info.consumer_port) + { + business->cur_vernode = vernode; + cJSON_AddNumberToObject(vernode->metajson, "version", vernode->version); + doris_config_mem_version_finish(NULL, business); + } + + business->version_cfgnum = vernode->total_cfgs; + doris_config_common_version_finish(business); + business->cfgver_head->latest_version = vernode->version; + + if(vernode->synctx != NULL) + { + doris_prod_upload_ctx_destroy(vernode->synctx); + vernode->synctx = NULL; + } + vernode->version_finished = 1; + business->posts_on_the_way--; + business->cur_vernode = NULL; // + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_POST_ON_THE_WAY], FS_OP_SET, business->posts_on_the_way); +} + +void http_config_direct_version_finish(struct version_list_node *vernode, struct evhttp_request *req, int64_t set_version) +{ + struct doris_business *business=vernode->business; + char version[32], token[64]; + int64_t new_version; + + if(evtimer_pending(&vernode->timer_expire, NULL)) + { + evtimer_del(&vernode->timer_expire); + } + + if(set_version == 0) + { + new_version = business->cfgver_head->latest_version + 1; + } + else + { + new_version = set_version; + } + sprintf(token, "%s", vernode->token); + doris_config_post_version_finish(business, vernode, new_version); + + sprintf(version, "%lu", new_version); + evhttp_add_header(evhttp_request_get_output_headers(req), "X-Set-Version", version); + evhttp_send_reply(req, 200, "OK", NULL); + + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, post server version finish, token: %s, version: %lu", business->bizname, token, new_version); +} + +void prod_sync_verend_result_cb(enum PROD_VEROP_RES result, int64_t version, void *userdata) +{ + struct version_list_node *vernode=(struct version_list_node *)userdata; + + vernode->retry_times++; + vernode->syncing = 0; + switch(result) + { + case VERSIONOP_RES_OK: + http_config_direct_version_finish(vernode, vernode->req, version); + break; + + case VERSIONOP_RES_ERROR: + evhttp_send_error(vernode->req, 500, "version end sync error res_code"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "\033[1;31;40mbusiness: %s, version end sync error res_code, abandon it. Send 500 response to client.\033[0m", vernode->business->bizname); + break; + + case VERSIONOP_CURL_ERROR: + if(atomic_read(&vernode->business->ready_to_sync) && (vernode->retry_times < 3)) + { + vernode->syncing = 1; + doris_prod_version_end(vernode->synctx, prod_sync_verend_result_cb, vernode); + } + else + { + http_config_direct_version_finish(vernode, vernode->req, 0); + business_set_sync_peer_abnormal(vernode->business); + } + break; + default: assert(0);break; + } +} + +void http_prod_server_verion_end_cb(struct evhttp_request *req, void *arg) +{ + struct doris_business *business=(struct doris_business *)arg; + struct version_list_node *vernode; + struct evkeyvalq params; + char version[32]; + + if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid"); + return; + } + if(NULL == (vernode = lookup_vernode_struct_from_name_renew(business, ¶ms))) + { + evhttp_clear_headers(¶ms); + evhttp_send_error(req, HTTP_NOTFOUND, "Parameter token invalid"); + return; + } + evhttp_clear_headers(¶ms); + + if(vernode->version_finished) + { + sprintf(version, "%lu", vernode->version); + evhttp_add_header(evhttp_request_get_output_headers(req), "X-Set-Version", version); + evhttp_send_error(req, HTTP_OK, "version already finished"); //֤һ + return; + } + if(vernode->cur_table != NULL || vernode->syncing) + { + evhttp_send_error(req, 300, "table not finished yet"); + return; + } + + if(!atomic_read(&business->ready_to_sync) || + NULL!=evhttp_find_header(evhttp_request_get_input_headers(req), "X-Doris-Master-Slave-Sync")) + { + return http_config_direct_version_finish(vernode, req, 0); + } + + if(vernode->synctx == NULL) + { + evhttp_send_error(req, 400, "illegal server host, cannt change server durain version life cycle"); + return; + } + vernode->retry_times = 0; + vernode->syncing = 1; + vernode->req = req; + doris_prod_version_end(vernode->synctx, prod_sync_verend_result_cb, vernode); +} + +static void post_vernode_expire_destroy_cb(int fd, short kind, void *userp) +{ + struct version_list_node *vernode=(struct version_list_node *)userp; + struct table_list_node *tablenode; + struct timeval tv; + + if(vernode->syncing) + { + tv.tv_sec = g_doris_server_info.post_vernode_ttl; + tv.tv_usec = 0; + evtimer_add(&vernode->timer_expire, &tv); + return; + } + if(vernode->synctx != NULL) + { + doris_prod_upload_ctx_destroy(vernode->synctx); + vernode->synctx = NULL; + } + + if(vernode->fp_idx_file != NULL) + { + fclose(vernode->fp_idx_file); + remove(vernode->tmp_index_path); + } + if(vernode->cur_table!=NULL && vernode->cur_table->fp_cfg_file != NULL) + { + fclose(vernode->cur_table->fp_cfg_file); + remove(vernode->cur_table->localpath); + } + TAILQ_FOREACH(tablenode, &vernode->table_head, table_node) + { + remove(tablenode->localpath); + } + vernode->business->posts_on_the_way--; + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_VERSION_EXPIRES], 0, FS_OP_ADD, 1); + FS_operate(g_doris_server_info.fsstat_handle, vernode->business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_POST_ON_THE_WAY], FS_OP_SET, vernode->business->posts_on_the_way); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, token %s expires", vernode->business->bizname, vernode->token); + config_version_node_cleanup(vernode); +} + +struct version_list_node *doris_config_post_version_prepare(struct doris_business *business, int32_t cfgtype) +{ + struct version_list_node *vernode; + struct timeval tv; + + vernode = (struct version_list_node *)calloc(1, sizeof(struct version_list_node)); + vernode->business = business; + vernode->cfg_type = cfgtype; + + if(business->concurrency_allowed) + { + tv.tv_sec = g_doris_server_info.post_vernode_ttl; + tv.tv_usec = 0; + evtimer_assign(&vernode->timer_expire, business->worker_evbase, post_vernode_expire_destroy_cb, vernode); + evtimer_add(&vernode->timer_expire, &tv); + } + return vernode; +} + +void doris_config_post_version_start(struct version_list_node *cur_vernode, const char *token) +{ + struct doris_business *business=cur_vernode->business; + + snprintf(cur_vernode->token, 64, "%s", token); + if(cur_vernode->cfg_type == CFG_UPDATE_TYPE_FULL) + { + snprintf(cur_vernode->tmp_index_path, 512, "%s/inc/full_config_index.%s.ing", business->store_path_root, token); + } + else + { + snprintf(cur_vernode->tmp_index_path, 512, "%s/inc/full_config_index.%s.ing", business->store_path_root, token); + } + if(NULL==(cur_vernode->fp_idx_file = fopen(cur_vernode->tmp_index_path, "w+"))) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, fopen %s failed: %s", business->bizname, cur_vernode->tmp_index_path, strerror(errno)); + assert(0); + } + + if(g_doris_server_info.consumer_port) + { + TAILQ_INIT(&cur_vernode->table_head); + cur_vernode->metajson = cJSON_CreateObject(); + cur_vernode->arrayjson= cJSON_CreateArray(); + cJSON_AddNumberToObject(cur_vernode->metajson, "type", cur_vernode->cfg_type); + } + business->token2node->insert(make_pair(string(token), cur_vernode)); +} + +void http_post_direct_version_start(struct version_list_node *cur_vernode, struct evhttp_request *req, const char *role) +{ + struct doris_business *business=cur_vernode->business; + char token[64], *p; + struct evbuffer *evbuf; + cJSON *meta; + + prod_server_generate_token(business, token, 64); + doris_config_post_version_start(cur_vernode, token); + + meta = cJSON_CreateObject(); + cJSON_AddStringToObject(meta, "token", token); + p = cJSON_PrintUnformatted(meta); + cJSON_Delete(meta); + + evbuf = evbuffer_new(); + evbuffer_add(evbuf, p, strlen(p)); + evhttp_send_reply(req, 200, "OK", evbuf); + evbuffer_free(evbuf); + cur_vernode->req = NULL; + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, post %s server send response version start: %s", business->bizname, role, p); + free(p); +} + +void try_restore_from_busy_peer(struct version_list_node *cur_vernode, const char *body, bool busy) +{ + struct doris_business *business=cur_vernode->business; + struct evbuffer *evbuf; + cJSON *meta, *token; + + /*Զ˼Ȼbusy˵һtokentokenIJǶԷ*/ + if((NULL==(meta=cJSON_Parse(body))) || NULL==(token=(cJSON_GetObjectItem(meta, "token")))) + { + assert(0); + } + /*һûϴãΪԷ;post servercurlʧܵ*/ + assert(NULL == cJSON_GetObjectItem(meta, "configs")); + + doris_config_post_version_start(cur_vernode, token->valuestring); + cJSON_Delete(meta); + + evbuf = evbuffer_new(); + evbuffer_add(evbuf, body, strlen(body)); + evhttp_send_reply(cur_vernode->req, 200, "OK", evbuf); + evbuffer_free(evbuf); + cur_vernode->req = NULL; + if(busy) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "\033[33mbusiness: %s, restore from busy peer, post master server send response version start: %s\033[0m", business->bizname, body); + } + else + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, post master server send response version start: %s", business->bizname, body); + } +} + +void prod_sync_verstart_result_cb(enum PROD_VERSTART_RES result, const char *body, void *userdata) +{ + struct version_list_node *vernode=(struct version_list_node *)userdata; + struct doris_business *business=vernode->business; + + vernode->retry_times++; + vernode->syncing = 0; + switch(result) + { + case VERSTART_RES_OK: + try_restore_from_busy_peer(vernode, body, false); + break; + + case VERSTART_RES_BUSY: //һǰCURLEģrate limit + try_restore_from_busy_peer(vernode, body, true); + break; + + case VERSTART_RES_ERROR: //ǷֱӷظClient + evhttp_send_error(vernode->req, 500, "version start sync error res_code"); + doris_prod_upload_ctx_destroy(vernode->synctx); + free(vernode); + business->cur_vernode = NULL; + business->posts_on_the_way--; + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_POST_ON_THE_WAY], FS_OP_SET, business->posts_on_the_way); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "\033[1;31;40mbusiness: %s, version start sync error res_code, abandon it. Send 500 response to client.\033[0m", business->bizname); + break; + + case VERSTART_CURL_ERROR: + if(atomic_read(&business->ready_to_sync) && (vernode->retry_times < 3)) + { + vernode->syncing = 1; + doris_prod_version_start_with_cb(vernode->synctx, prod_sync_verstart_result_cb, vernode); + } + else + { + http_post_direct_version_start(vernode, vernode->req, "master"); + business_set_sync_peer_abnormal(vernode->business); + } + break; + default: assert(0);break; + } +} + +void concurrency_send_busy_reply(struct doris_business *business, struct evhttp_request *req) +{ + char *p; + struct evbuffer *evbuf; + + /*version starṭͬδtoken;汾ʱܽ*/ + if(business->cur_vernode==NULL || business->cur_vernode->token[0]=='\0') + { + evhttp_send_error(req, 400, "another empty uploading busy"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_DEBUG, "business: %s busy starting, posts-on-the-way: %d", business->bizname, business->posts_on_the_way); + return; + } + + /*version start;汾ͬõԷtoken*/ + p = vernode_print_json_meta(business->cur_vernode); + evbuf = evbuffer_new(); + evbuffer_add(evbuf, p, strlen(p)); + evhttp_send_reply(req, 300, "another uploading busy", evbuf); + evbuffer_free(evbuf); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s busy, posts-on-the-way: %d, reply: %s", business->bizname, business->posts_on_the_way, p); + free(p); +} + +void http_prod_server_verion_start_cb(struct evhttp_request *req, void *arg) +{ + struct doris_business *argbiz=(struct doris_business *)arg, *business; + struct evkeyvalq params; + const char *type; + int cfgtype; + + if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid"); + return; + } + if(NULL == (business = lookup_bizstruct_from_name(¶ms)) || business!=argbiz) + { + evhttp_clear_headers(¶ms); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameter business invalid"); + return; + } + if(NULL == (type = evhttp_find_header(¶ms, "type")) || ((cfgtype=atoi(type))!=1 && cfgtype!=2)) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + evhttp_clear_headers(¶ms); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameter type invalid"); + return ; + } + evhttp_clear_headers(¶ms); + if(!business->concurrency_allowed && business->posts_on_the_way>0) + { + return concurrency_send_busy_reply(business, req); + } + if(business->posts_on_the_way > g_doris_server_info.max_concurrent_reqs) + { + evhttp_send_error(req, HTTP_SERVUNAVAIL, "Too many concurrent requests, service unavailable"); + return ; + } + + /*ڲbusiness->cur_vernodeʼղ䣻*/ + business->cur_vernode = doris_config_post_version_prepare(business, cfgtype); + business->posts_on_the_way++; + business->type = cfgtype; + FS_operate(g_doris_server_info.fsstat_handle, business->fs_lineid, g_doris_server_info.fsstat_column[DRS_FSCLM_POST_ON_THE_WAY], FS_OP_SET, business->posts_on_the_way); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s receives a version start request, posts-on-the-way: %d", business->bizname, business->posts_on_the_way); + + if(NULL != evhttp_find_header(evhttp_request_get_input_headers(req), "X-Doris-Master-Slave-Sync")) //ڲͬ + { + return http_post_direct_version_start(business->cur_vernode, req, "slave"); + } + + if(atomic_read(&business->ready_to_sync) && + (NULL!=(business->cur_vernode->synctx=doris_prod_upload_ctx_new(business->instance, business->bizname, cfgtype)))) + { + business->cur_vernode->retry_times = 0; + business->cur_vernode->req = req; + business->cur_vernode->syncing = 1; + doris_prod_version_start_with_cb(business->cur_vernode->synctx, prod_sync_verstart_result_cb, business->cur_vernode); + } + else + { + http_post_direct_version_start(business->cur_vernode, req, "master"); + business_set_sync_peer_abnormal(business); + } +} + +bool upload_frag_argument_check_offset(struct evhttp_request *req, struct evkeyvalq *params, + struct version_list_node *vernode, struct internal_tablemeta *tablemeta) +{ + const char *tmparg; + char *endptr=NULL, curoffset[32]; + size_t length; + + tablemeta->islast = 0; + if(NULL!=(tmparg=evhttp_find_header(params, "last")) && !strcasecmp(tmparg, "true")) + { + tablemeta->islast = 1; + } + if((length=evbuffer_get_length(evhttp_request_get_input_buffer(req))) > 0) + { + if(NULL == (tmparg = evhttp_find_header(params, "offset"))) + { + evhttp_send_error(req, 401, "Parameter offset not found"); + return false; + } + tablemeta->offset = strtol(tmparg, &endptr, 10); + if(*endptr != '\0') + { + evhttp_send_reply(req, 401, "Parameter offset invalid", NULL); + return false; + } + if(vernode->cur_table == NULL) + { + if(tablemeta->offset != 0) + { + evhttp_send_reply(req, 401, "Parameter offset is not starting from 0", NULL); + return false; + } + } + else if(tablemeta->offset+length <= vernode->cur_table->cur_totallen) + { + evhttp_send_reply(req, 201, "Parameter offset already uploaded", NULL); + return false; + } + else if(tablemeta->offset != vernode->cur_table->cur_totallen) + { + sprintf(curoffset, "%lu", vernode->cur_table->cur_totallen); + evhttp_add_header(evhttp_request_get_output_headers(req), "X-Current-Offset", curoffset); + evhttp_send_reply(req, 401, "Parameter offset invalid", NULL); + return false; + } + } + else if(!tablemeta->islast || vernode->cur_table==NULL) //lastδ˵δϴ + { + evhttp_send_error(req, 400, "Content length is zero, but parameter last!=true; or total length is zero, but parameter last=true"); + return false; + } + return true; +} + +struct version_list_node *upload_file_arguments_valid_check(struct evhttp_request *req, + struct doris_business *business, struct internal_tablemeta *tablemeta, bool fragcheck) +{ + struct evkeyvalq params; + struct version_list_node *vernode; + struct table_list_node *tablenode; + const char *tablename, *tmparg; + + if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid"); + return NULL; + } + if(NULL==(vernode = lookup_vernode_struct_from_name_renew(business, ¶ms))) + { + evhttp_send_error(req, HTTP_NOTFOUND, "Parameter token invalid"); + evhttp_clear_headers(¶ms); + return NULL; + } + if(NULL == (tablename = evhttp_find_header(¶ms, "tablename"))) + { + evhttp_send_error(req, HTTP_BADREQUEST, "Parameter tablename invalid"); + evhttp_clear_headers(¶ms); + return NULL; + } + + /*ϸδűϴ*/ + if(vernode->cur_table!=NULL && (vernode->syncing || strcmp(vernode->cur_table->tablename, tablename))) + { + evhttp_send_error(req, 300, "tablename busy"); + evhttp_clear_headers(¶ms); + return NULL; + } + /*finishedŻ鿴Ƿиñ֧ظı*/ + tablenode = TAILQ_FIRST(&vernode->table_head); + while(tablenode!=NULL && strcmp(tablename, tablenode->tablename)) + { + tablenode = TAILQ_NEXT(tablenode, table_node); + } + if(tablenode != NULL) + { + evhttp_send_error(req, HTTP_BADREQUEST, "tablename already finished"); + evhttp_clear_headers(¶ms); + return NULL; + } + + if(fragcheck && !upload_frag_argument_check_offset(req, ¶ms, vernode, tablemeta)) + { + evhttp_clear_headers(¶ms); + return NULL; + } + + snprintf(tablemeta->tablename, 64, "%s", tablename); + if(NULL == (tmparg = evhttp_find_header(¶ms, "filename"))) + { + tablemeta->filename[0] = '\0'; + } + else + { + snprintf(tablemeta->filename, 64, "%s", tmparg); + } + evhttp_clear_headers(¶ms); + return vernode; +} + +bool upload_frag_check_content_md5(struct evhttp_request *req, const char *content, size_t len, char *md5str, int md5size) +{ + const char *md5; + MD5_CTX ctx; + + if(NULL == (md5=evhttp_find_header(evhttp_request_get_input_headers(req), "Content-MD5"))) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + evhttp_send_error(req, 402, "Content-MD5 header not found"); + return false; + } + MD5_Init(&ctx); + MD5_Update(&ctx, content, len); + scandir_md5_final_string(&ctx, md5str, md5size); + if(strcasecmp(md5, md5str)) + { + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_CLIENT_INVALID_REQ], 0, FS_OP_ADD, 1); + evhttp_send_error(req, 402, "Content-MD5 not match"); + return false; + } + return true; +} + +void doris_config_post_cfgfile_prepare(struct version_list_node *cur_vernode, + struct internal_tablemeta *tablemeta, const char *md5, u_int32_t cfgnum, char *content, size_t size) +{ + if(cur_vernode->cur_table == NULL) + { + cur_vernode->cur_table = (struct table_list_node *)calloc(1, sizeof(struct table_list_node)); + cur_vernode->cur_table->cfgnum = cfgnum; + cur_vernode->total_cfgs += cfgnum; + sprintf(cur_vernode->cur_table->tablename, "%s", tablemeta->tablename); + if(tablemeta->filename[0] != '\0') //Clientָļ + { + sprintf(cur_vernode->cur_table->filename, "%s", tablemeta->filename); + } + else + { + snprintf(cur_vernode->cur_table->filename, 128, "%s.%s", tablemeta->tablename, cur_vernode->token); + } + MD5_Init(&cur_vernode->cur_table->md5ctx); + TAILQ_INIT(&cur_vernode->cur_table->frag_head); + } + cur_vernode->cur_table->fragcontent = content; + cur_vernode->cur_table->fragsize = size; + cur_vernode->cur_table->finished = tablemeta->islast; + sprintf(cur_vernode->cur_table->fragmd5, "%s", md5); + + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_DEBUG, "business: %s, table %s receives a file part, offset: %lu, size: %lu", + cur_vernode->business->bizname, tablemeta->tablename, tablemeta->offset, size); +} + +void doris_config_post_cfgfile_start(struct version_list_node *vernode, struct evhttp_request *req) +{ + struct tablemeta meta; + + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, table %s start...", vernode->business->bizname, vernode->cur_table->filename); + FS_operate(g_doris_server_info.fsstat_handle, g_doris_server_info.fsstat_field[DRS_FSSTAT_RECV_START_FILES], 0, FS_OP_ADD, 1); + + meta.tablename = vernode->cur_table->tablename; + meta.filename = vernode->cur_table->filename; + meta.userregion = evhttp_find_header(evhttp_request_get_input_headers(req), "X-User-Info"); + meta.cfgnum = vernode->cur_table->cfgnum; + meta.size = 0; + + vernode->business->type = vernode->cfg_type; + vernode->business->fp_idx_file = vernode->fp_idx_file; + doris_config_file_cfgfile_start(NULL, &meta, NULL, vernode->business); + sprintf(vernode->cur_table->localpath, "%s", vernode->business->cfg_file_path); + vernode->cur_table->fp_cfg_file = vernode->business->fp_cfg_file; + + if(g_doris_server_info.consumer_port) + { + vernode->cur_table->table_meta = cJSON_CreateObject(); + cJSON_AddStringToObject(vernode->cur_table->table_meta, "tablename", meta.tablename); + cJSON_AddStringToObject(vernode->cur_table->table_meta, "filename", meta.filename); + cJSON_AddNumberToObject(vernode->cur_table->table_meta, "cfg_num", meta.cfgnum); + if(meta.userregion != NULL) + { + cJSON_AddStringToObject(vernode->cur_table->table_meta, "user_region", meta.userregion); + } + } +} + +void doris_config_post_cfgfile_finish(struct version_list_node *vernode, const char *md5str) +{ + doris_config_common_cfgfile_finish(vernode->business); + fclose(vernode->cur_table->fp_cfg_file); + + assert(vernode->cur_table->filesize == 0); + vernode->cur_table->filesize = vernode->cur_table->cur_totallen; + if(g_doris_server_info.consumer_port) + { + cJSON_AddNumberToObject(vernode->cur_table->table_meta, "size", vernode->cur_table->filesize); + cJSON_AddStringToObject(vernode->cur_table->table_meta, "md5", md5str); + cJSON_AddItemToArray(vernode->arrayjson, vernode->cur_table->table_meta); + vernode->cur_table->table_meta = NULL; + if(vernode->cur_table->cur_frag != NULL) + { + if(vernode->cur_table->cur_frag->totalsize > vernode->cur_table->cur_frag->cur_fraglen) + { + char *content = (char *)malloc(vernode->cur_table->cur_frag->cur_fraglen); + memcpy(content, vernode->cur_table->cur_frag->content, vernode->cur_table->cur_frag->cur_fraglen); + free(vernode->cur_table->cur_frag->content); + vernode->cur_table->cur_frag->content = content; + vernode->cur_table->cur_frag->totalsize = vernode->cur_table->cur_frag->cur_fraglen; + vernode->cur_table->cur_frag->end = vernode->cur_table->filesize - 1; + } + TAILQ_INSERT_TAIL(&vernode->cur_table->frag_head, vernode->cur_table->cur_frag, frag_node); + assert(vernode->cur_table->cur_frag->cur_fraglen == vernode->cur_table->cur_frag->end - vernode->cur_table->cur_frag->start + 1); + vernode->cur_table->cur_frag = NULL; + } + } + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "business: %s, table %s finished", vernode->business->bizname, vernode->cur_table->filename); + TAILQ_INSERT_TAIL(&vernode->table_head, vernode->cur_table, table_node); + vernode->cur_table = NULL; //գ׼һű +} + +void http_config_direct_cfgfile_update(struct version_list_node *vernode, struct evhttp_request *req) +{ + size_t writen_len; + char md5str[40]; + + if(vernode->cur_table->cur_totallen == 0) //start + { + doris_config_post_cfgfile_start(vernode, req); + } + if(vernode->cur_table->fragsize > 0) + { + writen_len = fwrite(vernode->cur_table->fragcontent, 1, vernode->cur_table->fragsize, vernode->cur_table->fp_cfg_file); + if(writen_len != vernode->cur_table->fragsize) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "business: %s, fwrite %s failed: %s", vernode->business->bizname, vernode->cur_table->localpath, strerror(errno)); + assert(0); + } + if(g_doris_server_info.consumer_port) + { + vernode->business->cur_vernode = vernode; + doris_config_mem_cfgfile_update(NULL, vernode->cur_table->fragcontent, vernode->cur_table->fragsize, vernode->business); + } + else + { + vernode->cur_table->cur_totallen += vernode->cur_table->fragsize; + } + if(!vernode->cur_table->onceupload) + { + MD5_Update(&vernode->cur_table->md5ctx, vernode->cur_table->fragcontent, vernode->cur_table->fragsize); + } + free(vernode->cur_table->fragcontent); + } + if(vernode->cur_table->finished) //end + { + if(!vernode->cur_table->onceupload) + { + scandir_md5_final_string(&vernode->cur_table->md5ctx, md5str, 40); + doris_config_post_cfgfile_finish(vernode, md5str); + evhttp_add_header(evhttp_request_get_output_headers(req), "X-Content-MD5", md5str); + } + else + { + doris_config_post_cfgfile_finish(vernode, vernode->cur_table->fragmd5); + } + } + evhttp_send_reply(req, HTTP_OK, "OK", NULL); +} + +void prod_sync_upload_frag_cb(enum PROD_VEROP_RES result,void * userdata) +{ + struct version_list_node *vernode=(struct version_list_node *)userdata; + struct table_meta meta; + + vernode->retry_times++; + vernode->syncing = 0; + switch(result) + { + case VERSIONOP_RES_OK: + http_config_direct_cfgfile_update(vernode, vernode->req); + break; + + case VERSIONOP_RES_ERROR: + evhttp_send_error(vernode->req, 500, "frag sync error res_code"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "\033[1;31;40mbusiness: %s, frag sync error res_code, abandon it. Send 500 response to client.\033[0m", vernode->business->bizname); + break; + + case VERSIONOP_CURL_ERROR: + if(atomic_read(&vernode->business->ready_to_sync) && (vernode->retry_times < 3)) + { + vernode->syncing = 1; + meta.md5 = vernode->cur_table->fragmd5; + meta.cfgnum = vernode->cur_table->cfgnum; + meta.tablename = vernode->cur_table->tablename; + meta.filename = vernode->cur_table->filename; + meta.userregion = evhttp_find_header(evhttp_request_get_input_headers(vernode->req), "X-User-Info"); + if(vernode->cur_table->onceupload) + { + doris_prod_upload_once_with_cb(vernode->synctx, vernode->cur_table->fragcontent, + vernode->cur_table->fragsize, &meta, prod_sync_upload_frag_cb, vernode); + } + else + { + doris_prod_upload_frag_with_cb(vernode->synctx, vernode->cur_table->fragcontent, vernode->cur_table->fragsize, vernode->cur_table->cur_totallen, + vernode->cur_table->finished?true:false, &meta, prod_sync_upload_frag_cb, vernode); + } + } + else + { + http_config_direct_cfgfile_update(vernode, vernode->req); + business_set_sync_peer_abnormal(vernode->business); + } + break; + default: assert(0);break; + } +} + +void http_prod_server_file_once_cb(struct evhttp_request *req, void *arg) +{ + struct doris_business *business=(struct doris_business *)arg; + struct version_list_node *vernode; + char *content, md5str[64]; + const char *tmp; + struct internal_tablemeta tablemeta; + size_t size; + struct table_meta meta; + int32_t cfgnum=0, need_sync=0; + + if(NULL == (vernode=upload_file_arguments_valid_check(req, business, &tablemeta, false))) + { + return; + } + tablemeta.islast = 1; + tablemeta.offset = 0; + + /*ԭʼClientX-Doris-Master-Slave-Syncͷͬһ汾*/ + if(atomic_read(&business->ready_to_sync) && + NULL==evhttp_find_header(evhttp_request_get_input_headers(req), "X-Doris-Master-Slave-Sync")) + { + need_sync = 1; + } + if(need_sync && vernode->synctx==NULL) + { + evhttp_send_error(req, 400, "illegal server host, cannt change server durain version life cycle"); + return ; + } + + if((size=evbuffer_get_length(evhttp_request_get_input_buffer(req))) == 0) + { + evhttp_send_error(req, 400, "no content"); + return ; + } + content = (char*)malloc(size); + if(size != (size_t)evbuffer_copyout(evhttp_request_get_input_buffer(req), content, size)) + { + assert(0); + } + if(!upload_frag_check_content_md5(req, content, size, md5str, 64)) + { + free(content); + return ; + } + + if(NULL != (tmp=evhttp_find_header(evhttp_request_get_input_headers(req), "X-Config-Num"))) + { + cfgnum = atoi(tmp);; + } + doris_config_post_cfgfile_prepare(vernode, &tablemeta, md5str, cfgnum, content, size); + + meta.md5 = md5str; + meta.cfgnum = cfgnum; + meta.tablename = tablemeta.tablename; + meta.userregion = evhttp_find_header(evhttp_request_get_input_headers(req), "X-User-Info"); + meta.filename = vernode->cur_table->filename; + vernode->cur_table->onceupload = true; + + if(!need_sync) + { + return http_config_direct_cfgfile_update(vernode, req); + } + vernode->retry_times = 0; + vernode->req = req; + vernode->syncing = 1; + doris_prod_upload_once_with_cb(vernode->synctx, content, size, &meta, prod_sync_upload_frag_cb, vernode); +} + +void http_prod_server_file_frag_cb(struct evhttp_request *req, void *arg) +{ + struct doris_business *business=(struct doris_business *)arg; + struct version_list_node *vernode; + char *content=NULL, md5str[64]; + const char *tmp; + struct internal_tablemeta tablemeta; + size_t size=0; + struct table_meta meta; + int32_t cfgnum=0, need_sync=0; + + if(NULL == (vernode=upload_file_arguments_valid_check(req, business, &tablemeta, true))) + { + return; + } + if((size=evbuffer_get_length(evhttp_request_get_input_buffer(req)))==0 && !tablemeta.islast) + { + evhttp_send_error(req, 400, "no content"); + return ; + } + + if(atomic_read(&business->ready_to_sync) && + NULL==evhttp_find_header(evhttp_request_get_input_headers(req), "X-Doris-Master-Slave-Sync")) + { + need_sync = 1; + } + if(need_sync && vernode->synctx==NULL) + { + evhttp_send_error(req, 400, "illegal server host, cannt change server durain version life cycle"); + return ; + } + + if(size > 0) + { + content = (char *)malloc(size); + if(size != (size_t)evbuffer_copyout(evhttp_request_get_input_buffer(req), content, size)) + { + assert(0); + } + if(!upload_frag_check_content_md5(req, content, size, md5str, 64)) + { + free(content); + return ; + } + } + + if(NULL != (tmp=evhttp_find_header(evhttp_request_get_input_headers(req), "X-Config-Num"))) + { + cfgnum = atoi(tmp);; + } + doris_config_post_cfgfile_prepare(vernode, &tablemeta, md5str, cfgnum, content, size); + + meta.md5 = md5str; + meta.cfgnum = cfgnum; + meta.tablename = tablemeta.tablename; + meta.userregion = evhttp_find_header(evhttp_request_get_input_headers(req), "X-User-Info"); + meta.filename = vernode->cur_table->filename; + if(tablemeta.islast && tablemeta.offset==0) + { + vernode->cur_table->onceupload = true; + } + + if(!need_sync) + { + return http_config_direct_cfgfile_update(vernode, req); + } + vernode->retry_times = 0; + vernode->req = req; + vernode->syncing = 1; + doris_prod_upload_frag_with_cb(vernode->synctx, content, size, vernode->cur_table->cur_totallen, + tablemeta.islast?true:false, &meta, prod_sync_upload_frag_cb, vernode); +} + +void start_business_http_post_server(struct doris_business *business) +{ + struct evhttp *worker_http; + + if((business->listener_prod = doris_create_listen_socket(business->producer_port)) < 0) + { + assert(0);return; + } + business->source_from = RECV_WAY_HTTP_POST; + worker_http = evhttp_new(business->worker_evbase); + if(g_doris_server_info.ssl_conn_on) + { + evhttp_set_bevcb(worker_http, doris_https_bufferevent_cb, g_doris_server_info.ssl_instance); + } + + evhttp_set_cb(worker_http, "/version/start", http_prod_server_verion_start_cb, business); + evhttp_set_cb(worker_http, "/version/finish", http_prod_server_verion_end_cb, business); + evhttp_set_cb(worker_http, "/version/cancel", http_prod_server_verion_cancel_cb, business); + evhttp_set_cb(worker_http, "/version/check", http_prod_server_verion_check_cb, business); + evhttp_set_cb(worker_http, "/fileonce/upload", http_prod_server_file_once_cb, business); + evhttp_set_cb(worker_http, "/filefrag/upload", http_prod_server_file_frag_cb, business); + evhttp_set_allowed_methods(worker_http, EVHTTP_REQ_POST|EVHTTP_REQ_PUT|EVHTTP_REQ_HEAD); + evhttp_set_max_body_size(worker_http, g_doris_server_info.max_http_body_size); + + if(evhttp_accept_socket(worker_http, business->listener_prod)) + { + printf("evhttp_accept_socket %d error!\n", business->listener_prod); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "evhttp_accept_socket %d error!\n", business->listener_prod); + assert(0); + } +} + +void doris_config_version_sync_updated(struct doris_csum_instance *instance, void *userdata) +{ + struct doris_business *business=(struct doris_business *)userdata; + struct doris_csum_param *param; + u_int32_t references, business_post_ups; + + /*consuemerͬʱȷִֻһ*/ + param = doris_csum_instance_get_param(instance); + doris_csum_instance_destroy(instance); + references = doris_csum_param_get_refernces(param); + if(references == 0) + { + doris_csum_parameter_destroy(param); + } + + /*init sync instance*/ + business->instance = doris_prod_instance_new(business->param_prod, business->worker_evbase, g_doris_server_info.log_runtime); + if(business->instance == NULL) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "doris_prod_instance_new for %s failed", business->bizname); + assert(0);return; + } + + /*start worker*/ + start_business_http_post_server(business); + + /*ͬɣʾ汾serverһ(עһ£DZ, ʱserver᷵304)*/ + atomic_set(&business->ready_to_sync, 1); + + pthread_mutex_lock(&g_doris_server_info.mutex_lock); + business_post_ups = ++g_doris_server_info.business_post_ups; + pthread_mutex_unlock(&g_doris_server_info.mutex_lock); + if(business_post_ups == g_doris_server_info.business_post_num) + { + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mmid_post_server, MONITOR_VALUE_SET, PROMETHUES_POST_SERVER_OK); + } + else + { + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mmid_post_server, MONITOR_VALUE_SET, PROMETHUES_POST_SERVER_UPING); + } + assert(business_post_ups <= g_doris_server_info.business_post_num); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "\033[32m******Doris Producer worker for %s starts******\033[0m", business->bizname); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "HttpProducer, doris ready to sync for business: %s\n", business->bizname); +} + +/*thread_doris_client_recv_cfgversion_updated*/ +void* thread_http_post_recv_cfg(void *arg) +{ + struct doris_business *business=(struct doris_business *)arg; + struct event_base *client_evbase; + struct doris_csum_instance *instance; + struct doris_callbacks doris_cbs; + struct doris_arguments doris_args; + struct doris_idxfile_scanner *scanner; + enum DORIS_UPDATE_TYPE update_type; + char stored_path[512]; + + prctl(PR_SET_NAME, "http_post"); + + client_evbase = event_base_new(); + + business->source_from = RECV_WAY_IDX_FILE; + business->worker_evbase = client_evbase; + + scanner = doris_index_file_scanner(0); + + /*Retaive latest config to memory from Stored configs*/ + doris_cbs.version_start = doris_config_localmem_version_start; + doris_cbs.version_finish = doris_config_localmem_version_finish; + doris_cbs.version_error = doris_config_localmem_version_error; + doris_cbs.cfgfile_start = doris_config_localmem_cfgfile_start; + doris_cbs.cfgfile_update = doris_config_localmem_cfgfile_update; + doris_cbs.cfgfile_finish = doris_config_localmem_cfgfile_finish; + doris_cbs.version_updated= NULL; + doris_cbs.userdata = business; + + snprintf(stored_path, 512, "%s/full/index", business->store_path_root); + update_type = doris_index_file_traverse(scanner, stored_path, &doris_cbs, NULL, g_doris_server_info.log_runtime); + snprintf(stored_path, 512, "%s/inc/index", business->store_path_root); + do { + update_type = doris_index_file_traverse(scanner, stored_path, &doris_cbs, NULL, g_doris_server_info.log_runtime); + }while(update_type != CFG_UPDATE_TYPE_NONE); + + if(g_doris_server_info.cluster_sync_mode) /*Check new configs*/ + { + doris_cbs.version_start = doris_config_version_start; + doris_cbs.version_finish = doris_config_version_finish; + doris_cbs.version_error = doris_config_version_error; + doris_cbs.cfgfile_start = doris_config_cfgfile_start; + doris_cbs.cfgfile_update = doris_config_cfgfile_update; + doris_cbs.cfgfile_finish = doris_config_cfgfile_finish; + doris_cbs.version_updated= doris_config_version_sync_updated; + + business->source_from = RECV_WAY_DRS_CLIENT; + memset(&doris_args, 0, sizeof(struct doris_arguments)); + doris_args.current_version = scanner->cur_version; + sprintf(doris_args.bizname, "%s", business->bizname); + instance = doris_csum_instance_new(business->param_csum, client_evbase, &doris_cbs, &doris_args, g_doris_server_info.log_runtime); + if(instance == NULL) + { + assert(0);return NULL; + } + } + else + { + start_business_http_post_server(business); + } + + event_base_dispatch(client_evbase); + printf("Libevent dispath error, should not run here.\n"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Libevent dispath error, should not run here."); + assert(0);return NULL; +} + diff --git a/server/doris_server_receive.h b/server/doris_server_receive.h index 846ae78..70b4ea6 100644 --- a/server/doris_server_receive.h +++ b/server/doris_server_receive.h @@ -4,8 +4,23 @@ #include #include #include +#include #include +#include + +#include "doris_producer_client.h" + +#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 410) +#define atomic_inc(x) __sync_add_and_fetch((x),1) +#define atomic_dec(x) __sync_sub_and_fetch((x),1) +#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) +#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) +#define atomic_read(x) __sync_add_and_fetch((x),0) +#define atomic_set(x,y) __sync_lock_test_and_set((x),y) +#else +#error "GCC version should be 4.1.2 later" +#endif #include using namespace std; @@ -20,8 +35,10 @@ enum DORIS_SERVER_FS_FILED DRS_FSSTAT_CLIENT_META_REQ, DRS_FSSTAT_SEND_META_NONEW, DRS_FSSTAT_CLIENT_FILE_REQ, + DRS_FSSTAT_SEND_FILES, DRS_FSSTAT_SEND_FILE_BYTES, DRS_FSSTAT_SEND_FILE_RES_404, + DRS_FSSTAT_VERSION_EXPIRES, DRS_FSSTAT_FIELD_NUM, }; @@ -31,8 +48,8 @@ enum DORIS_SERVER_FS_COLUMN DRS_FSCLM_RECV_FULL_VER=0, DRS_FSCLM_RECV_INC_VER, DRS_FSCLM_RECV_FILES, + DRS_FSCLM_POST_ON_THE_WAY, DRS_FSCLM_SEND_META_RES, - DRS_FSCLM_SEND_FILE_RES, DRS_FSCLM_CUR_FULL_VERSION, DRS_FSCLM_CUR_INC_VERSION, DRS_FSCLM_CONFIG_TOTAL_NUM, @@ -58,6 +75,14 @@ struct cont_frag_node TAILQ_ENTRY(cont_frag_node) frag_node; }; +struct internal_tablemeta +{ + char tablename[64]; + char filename[64]; + size_t offset; + int32_t islast; +}; + struct table_list_node { char tablename[64]; @@ -65,20 +90,49 @@ struct table_list_node size_t filesize; size_t cur_totallen; + /*this part for http post server*/ + bool onceupload; + int32_t finished; + MD5_CTX md5ctx; + char tmppath[256]; + char filename[128]; + char fragmd5[36]; + u_int32_t cfgnum; + char *fragcontent; + size_t fragsize; + cJSON *table_meta; + + FILE *fp_cfg_file; + struct cont_frag_node *cur_frag; TAILQ_HEAD(__table_cont_node, cont_frag_node) frag_head; TAILQ_ENTRY(table_list_node) table_node; }; +struct doris_business; struct version_list_node { int64_t version; char *metacont; int32_t metalen; int16_t cfg_type; //1-full, 2-inc - int16_t cont_in_disk; + int8_t cont_in_disk; + int8_t version_finished; cJSON *metajson, *arrayjson; cJSON *table_meta; + /*this part for http post server*/ + FILE *fp_idx_file; + char token[64]; + char tmp_index_path[256]; //incĿ¼µȫʱÿ汾һ + int16_t retry_times; + int16_t syncing; + int32_t total_cfgs; + struct doris_business *business; + struct evhttp_request *req; + struct doris_upload_ctx *synctx; + struct table_list_node *cur_table; + struct event timer_expire; + TAILQ_HEAD(__table_list_node, table_list_node) table_head; TAILQ_ENTRY(version_list_node) version_node; }; @@ -95,35 +149,21 @@ struct version_list_handle struct version_list_handle *config_version_handle_new(void); -struct doris_business; -struct confile_save -{ - struct event_base *evbase; - struct doris_business *business; - int64_t version; - int32_t source_from; - int32_t type; - int64_t version_cfgnum; - char inc_index_path[256]; //incĿ¼µȫ - char tmp_index_path[256]; //incĿ¼µȫ - char full_index_path[256]; //fullĿ¼µȫ - char cfg_file_path[256]; - FILE *fp_cfg_file; - FILE *fp_idx_file; - - struct version_list_node *cur_vernode; - struct table_list_node *cur_table; - struct cont_frag_node *cur_frag; -}; - struct common_timer_event { struct event timer_event; void *data; }; +struct doris_business; +struct bufferevent *doris_https_bufferevent_cb(struct event_base *evabse, void *arg); +struct doris_business *lookup_bizstruct_from_name(const struct evkeyvalq *params); +void business_set_sync_peer_abnormal(struct doris_business *business); +void business_resume_sync_peer_normal(struct doris_business *business); + void* thread_doris_client_recv_cfg(void *arg); void* thread_index_file_recv_cfg(void *arg); +void* thread_http_post_recv_cfg(void *arg); #endif diff --git a/server/doris_server_scandir.cpp b/server/doris_server_scandir.cpp index 8840b64..a9e4404 100644 --- a/server/doris_server_scandir.cpp +++ b/server/doris_server_scandir.cpp @@ -17,7 +17,7 @@ #define MESA_RUNTIME_LOGV4(handle, lv, fmt, args...) \ MESA_handle_runtime_log((handle), (lv), "DorisServer", "%s:%d, " fmt, __FILENAME__, __LINE__, ##args) -static int scandir_md5_final_string(MD5_CTX *c, char *result, unsigned int size) +int scandir_md5_final_string(MD5_CTX *c, char *result, unsigned int size) { unsigned char md5[17]={0}; int i; @@ -202,12 +202,12 @@ int cm_read_cfg_index_file(const char* path, struct cfg_table_info* idx/*OUT*/, memset(line, 0, sizeof(line)); fgets(line, sizeof(line), fp); ret=sscanf(line,"%[^ \t]%*[ \t]%d%*[ \t]%s%*[ \t]%s", idx[i].table_name, &(idx[i].cfg_num), idx[i].cfg_path, idx[i].user_region); - if((ret!=3 && ret!=4) || idx[i].cfg_num==0)//jump over empty line + if((ret!=3 && ret!=4))//jump over empty line { continue; } ret=stat(idx[i].cfg_path, &file_info); - if(ret!=0) + if(ret!=0 || file_info.st_size==0) //ļ· { MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "%s of %s not exisit", idx[i].cfg_path, path); fclose(fp); @@ -315,12 +315,14 @@ enum DORIS_UPDATE_TYPE doris_index_file_traverse(struct doris_idxfile_scanner *s { MESA_RUNTIME_LOGV4(logger, RLOG_LV_INFO, "load %s", idx_path_array[i].path); table_num=cm_read_cfg_index_file(idx_path_array[i].path, table_array, CM_MAX_TABLE_NUM, logger); - if(table_num<0) + if(table_num<=0) { - MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "load %s faild, abandon udpate.", idx_path_array[i].path); + MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "\033[1;31;40mAlert! Load %s failed, skip this wrong version!!!!\033[0m\n", idx_path_array[i].path); update_type = CFG_UPDATE_TYPE_ERR; + scanner->cur_version = idx_path_array[i].version; //İ汾 break; } + scanner->cur_version = idx_path_array[i].version; cJSON *meta = doris_index_version_start(idx_path_array[i].version, table_array, table_num, update_type, doris_cbs); for(j=0; jcur_version = idx_path_array[i].version; doris_cbs->version_finish(NULL, doris_cbs->userdata); } else @@ -337,6 +338,7 @@ enum DORIS_UPDATE_TYPE doris_index_file_traverse(struct doris_idxfile_scanner *s update_type = CFG_UPDATE_TYPE_ERR; doris_cbs->version_error(NULL, doris_cbs->userdata); cJSON_Delete(meta); + MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "\033[1;31;40mAlert! Load %s failed, skip this wrong version!!!!\033[0m\n", idx_path_array[i].path); break; } cJSON_Delete(meta); diff --git a/server/doris_server_scandir.h b/server/doris_server_scandir.h index ab8d993..c9e68eb 100644 --- a/server/doris_server_scandir.h +++ b/server/doris_server_scandir.h @@ -1,7 +1,7 @@ #ifndef __DORIS_SERVER_SCANDIR_H__ #define __DORIS_SERVER_SCANDIR_H__ -#include "doris_client.h" +#include "doris_consumer_client.h" #define CM_MAX_TABLE_NUM 256 #define MAX_CONFIG_FN_LEN 256 @@ -45,6 +45,8 @@ struct doris_idxfile_scanner char oncebuf[ONCE_BUF_SIZE]; }; +int scandir_md5_final_string(MD5_CTX *c, char *result, unsigned int size); + struct doris_idxfile_scanner *doris_index_file_scanner(int64_t start_version); enum DORIS_UPDATE_TYPE doris_index_file_traverse(struct doris_idxfile_scanner *scanner, const char*idx_dir, struct doris_callbacks *doris_cbs, const char* dec_key, void* logger);