commit 26b1a0850061a6fad963772991abcd6303cd50f3 Author: linuxrc@163.com Date: Fri Jul 16 16:06:59 2021 +0800 鍒涘缓 diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..26dcb39 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.9) +project (doris) +#set (nirvana_platform_VERSION_MAJOR 1) +#set (nirvana_platform_VERSION_MINOR 1) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_C_STANDARD 11) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +add_definitions(-D_GNU_SOURCE) + +include_directories(${PROJECT_SOURCE_DIR}/include) + +set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/out) + +enable_testing() +add_subdirectory (support) +add_subdirectory (client) +add_subdirectory (server) + diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt new file mode 100644 index 0000000..e5beafe --- /dev/null +++ b/client/CMakeLists.txt @@ -0,0 +1,31 @@ +set (DORIS_CLIENT_SRC doris_client_fetch.cpp doris_client_http.cpp doris_client_transfer.cpp nirvana_conhash.cpp nirvana_murmurhash.cpp) + +add_definitions(-fPIC -Wall -g) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_CURRENT_SOURCE_DIR}/,,$(abspath $<))\"'") + +add_library(doris_client_static STATIC ${DORIS_CLIENT_SRC}) +set_target_properties(doris_client_static PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(doris_client_static PROPERTIES OUTPUT_NAME doris_client) +set_target_properties(doris_client_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) +target_link_libraries(doris_client_static libevent-static libcurl-static libevent-openssl-static openssl-ssl-static openssl-crypto-static cjson) +target_link_libraries(doris_client_static MESA_handle_logger MESA_prof_load MESA_field_stat2 pthread) +target_include_directories(doris_client_static PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_include_directories(doris_client_static PUBLIC ${PROJECT_SOURCE_DIR}/include) +set_property(TARGET doris_client_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/include) + +add_library(doris_client_dynamic SHARED ${DORIS_CLIENT_SRC}) +set_target_properties(doris_client_dynamic PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(doris_client_dynamic PROPERTIES OUTPUT_NAME doris_client) +set_target_properties(doris_client_dynamic PROPERTIES CLEAN_DIRECT_OUTPUT 1) +target_link_libraries(doris_client_dynamic libevent-dynamic libcurl-dynamic libevent-openssl-dynamic openssl-ssl-dynamic openssl-crypto-dynamic cjson-dynamic) +target_link_libraries(doris_client_dynamic MESA_handle_logger MESA_prof_load MESA_field_stat2 pthread) +target_include_directories(doris_client_dynamic PUBLIC ${PROJECT_SOURCE_DIR}/include) +target_include_directories(doris_client_dynamic PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) +set_property(TARGET doris_client_dynamic PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/include) + +#INSTALL (TARGETS doris_client_static doris_client_dynamic +# LIBRARY DESTINATION lib +# ARCHIVE DESTINATION lib) +#INSTALL (DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION include) +#INSTALL (FILES doris_client_threads.h doris_conhash.h doris_murmurhash.h DESTINATION include) diff --git a/client/doris_client_fetch.cpp b/client/doris_client_fetch.cpp new file mode 100644 index 0000000..50ed19b --- /dev/null +++ b/client/doris_client_fetch.cpp @@ -0,0 +1,589 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "doris_client_fetch.h" + +void easy_string_destroy(struct easy_string *estr) +{ + if(estr->buff != NULL) + { + free(estr->buff); + estr->buff = NULL; + estr->len = estr->size = 0; + } +} + +void easy_string_savedata(struct easy_string *estr, const char *data, size_t len) +{ + if(estr->size-estr->len < len+1) + { + estr->size += len*4+1; + estr->buff = (char*)realloc(estr->buff, estr->size); + } + + memcpy(estr->buff+estr->len, data, len); + estr->len += len; + estr->buff[estr->len]='\0'; +} + +void doris_confile_ctx_reset(struct doris_confile_ctx *ctx) +{ + struct doris_http_ctx *httpctx=ctx->httpctx; + memset(ctx, 0, sizeof(struct doris_confile_ctx)); + ctx->httpctx = httpctx; +} + +void doris_confile_ctx_destry(struct doris_confile_ctx *ctx) +{ + doris_confile_ctx_reset(ctx); + doris_http_ctx_destroy(ctx->httpctx); + ctx->httpctx = NULL; +} + +void doris_update_new_version(struct doris_instance *instance) +{ + instance->cur_version = instance->new_version; + instance->new_version += 1; +} + +void doris_request_restart_timer(struct doris_instance *instance, time_t wait_s) +{ + struct timeval tv; + + tv.tv_sec = wait_s; + tv.tv_usec = 0; + event_add(&instance->timer_fetch, &tv); +} + +void doris_fetch_next_confile_meta(struct doris_instance *instance) +{ + cJSON *cur_a_item, *sub; + + memset(&instance->curmeta, 0, sizeof(struct fetch_file_meta)); + + cur_a_item = cJSON_GetArrayItem(instance->array, instance->array_index); + instance->array_index++; + + sub = cJSON_GetObjectItem(cur_a_item, "tablename"); + instance->curmeta.table_name = sub->valuestring; + + sub = cJSON_GetObjectItem(cur_a_item, "size"); + instance->curmeta.size = sub->valuedouble; + + sub = cJSON_GetObjectItem(cur_a_item, "cfg_num"); + instance->curmeta.cfg_num = sub->valueint; +} + +void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code, long res_code, void *userp) +{ + struct doris_instance *instance = (struct doris_instance *)userp; + const char *pos_colon; + size_t datalen; + char buffer[64]; + int ret; + + if(instance->ctx.res_code == 0) //check code only once + { + instance->ctx.res_code = res_code; + assert(res_code != 0); + + if(res_code != 200 && res_code!=206) + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "Fetch confile %s failed, req_version=%lu, curlcode = %d", + instance->curmeta.table_name, instance->new_version, code); + return; + } + instance->retry_times = 0; + if(instance->curmeta.curoffset == 0) + { + instance->param->cbs.cfgfile_start(instance, instance->curmeta.table_name, + instance->curmeta.size, instance->curmeta.cfg_num, instance->param->cbs.userdata); + } + } + + if((pos_colon=(const char*)memchr(start, ':', bytes)) == NULL) + { + return ; + } + datalen = pos_colon - start; + switch(datalen) + { + case 14: + if(!strncasecmp(start, "Content-Length:", 15)) + { + memcpy(buffer, start+15, bytes-15); + buffer[bytes-15] = '\0'; + instance->ctx.contlength = atol(buffer); + } + break; + case 13: + if(!strncasecmp(start, "Content-Range:", 14)) + { + memcpy(buffer, start+13, bytes-13); + buffer[bytes-13] = '\0'; + ret = sscanf(buffer, "%*[^0-9]%lu-%lu/%lu", &instance->ctx.contl_start, &instance->ctx.contl_end, &instance->ctx.contl_total); + assert(ret == 3 && instance->ctx.contl_total == instance->curmeta.size && instance->ctx.contl_start==instance->curmeta.curoffset); + } + break; + default: break; + } +} + +void doris_http_confile_body_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp) +{ + struct doris_instance *instance = (struct doris_instance *)userp; + + if(code!=CURLE_OK || (instance->ctx.res_code!=200 && instance->ctx.res_code!=206) || (res_code!=200 && res_code!=206)) + { + return; + } + + instance->param->cbs.cfgfile_update(instance, ptr, bytes, instance->param->cbs.userdata); + instance->curmeta.curoffset += bytes; + instance->statistic.field[DRS_FS_FILED_RES_BYTES] += bytes; +} + +void doris_http_fetch_confile(struct doris_instance *instance); +void doris_http_confile_done_cb(CURLcode res, long res_code, const char *err, void *userp) +{ + struct doris_instance *instance = (struct doris_instance *)userp; + + if(instance->ctx.res_code != 200 && instance->ctx.res_code!=206) + { + goto out_error; + } + + if(res!=CURLE_OK || (res_code!=200 && res_code!=206)) + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Fetch confile %s failed, req_version=%lu, curlcode = %d, error: %s", + instance->curmeta.table_name, instance->new_version, res_code, err); + goto out_error; + } + if(instance->ctx.contl_total != 0) + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Fetch confile %s success, req_version=%lu, Content-Range: %lu-%lu/%lu", + instance->curmeta.table_name, instance->new_version, instance->ctx.contl_start, instance->ctx.contl_end, instance->ctx.contl_total); + } + else + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Fetch confile %s success, req_version=%lu, Content-Length: %lu/%lu", + instance->curmeta.table_name, instance->new_version, instance->ctx.contlength, instance->curmeta.size); + } + + instance->statistic.field[DRS_FS_FILED_RES_FRAGS] += 1; + if(instance->curmeta.curoffset >= instance->curmeta.size) //该文件下载完毕 + { + instance->statistic.field[DRS_FS_FILED_RES_FILES] += 1; + instance->param->cbs.cfgfile_finish(instance, instance->param->cbs.userdata); + if(instance->array_index == instance->array_size) + { + instance->param->cbs.version_finish(instance, instance->param->cbs.userdata); + instance->status = FETCH_STATUS_META; + doris_update_new_version(instance); + cJSON_Delete(instance->meta); + doris_confile_ctx_destry(&instance->ctx); + doris_request_restart_timer(instance, 0); + } + else + { + doris_fetch_next_confile_meta(instance); + doris_http_fetch_confile(instance); + } + } + else + { + doris_http_fetch_confile(instance); + } + return; + +out_error: + instance->statistic.field[DRS_FS_FILED_RES_FRAGERR] += 1; + if(instance->ctx.res_code == 404) //404应答的重新开始请求 + { + instance->retry_times = instance->param->fetch_max_tries; + } + else + { + instance->retry_times++; + } + if(instance->retry_times >= instance->param->fetch_max_tries) + { + instance->statistic.field[DRS_FS_FILED_RES_VERERR] += 1; + instance->param->cbs.version_error(instance, instance->param->cbs.userdata); + instance->retry_times = 0; + instance->status = FETCH_STATUS_META; + cJSON_Delete(instance->meta); + doris_confile_ctx_destry(&instance->ctx); + } + doris_request_restart_timer(instance, instance->param->retry_interval); +} + +void doris_http_fetch_confile(struct doris_instance *instance) +{ + struct doris_http_callback curlcbs; + char metauri[128], range[64]={0}; + + curlcbs.header_cb = doris_http_confile_header_cb; + curlcbs.write_cb = doris_http_confile_body_cb; + curlcbs.transfer_done_cb = doris_http_confile_done_cb; + curlcbs.userp = instance; + doris_confile_ctx_reset(&instance->ctx); + doris_http_ctx_reset(instance->ctx.httpctx, &curlcbs); + + //超大文件分段下载;上次未完成的继续下载 + if((instance->curmeta.size > instance->param->fetch_frag_size) || instance->curmeta.curoffset!=0) + { + sprintf(range, "Range: bytes=%lu-%lu", instance->curmeta.curoffset, instance->curmeta.curoffset + instance->param->fetch_frag_size - 1); + doris_http_ctx_add_header(instance->ctx.httpctx, range); + } + + snprintf(metauri, 128, "configfile?tablename=%s&version=%lu&businessid=%u", instance->curmeta.table_name, instance->new_version, instance->param->args.businessid); + if(doris_http_launch_get_request(instance->ctx.httpctx, metauri)) + { + instance->statistic.field[DRS_FS_FILED_REQ_FAIL] += 1; + doris_request_restart_timer(instance, instance->param->retry_interval); + } + else + { + instance->statistic.field[DRS_FS_FILED_REQ_FILES] += 1; + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Launch confile %s GET, req_version=%lu, %s", + instance->curmeta.table_name, instance->new_version, range); + } +} + +void doris_http_meta_header_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp) +{ + struct doris_instance *instance = (struct doris_instance *)userp; + + //check code only once + if(instance->ctx.res_code != 0) + { + return; + } + instance->ctx.res_code = res_code; + assert(res_code != 0); + + if(res_code != 200) + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "No new meta found, cur_version=%lu, req_version=%lu, curlcode = %d", + instance->cur_version, instance->new_version, code); + } +} + +void doris_http_meta_body_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp) +{ + struct doris_instance *instance = (struct doris_instance *)userp; + + if(code!=CURLE_OK || res_code!=200) + { + return; + } + + easy_string_savedata(&instance->estr, (const char*)ptr, bytes); +} + +void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void *userp) +{ + struct doris_instance *instance = (struct doris_instance *)userp; + + if(instance->ctx.res_code != 200) + { + goto out_error; + } + + if(res!=CURLE_OK || res_code!=200) + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "No new meta found, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s", + instance->cur_version, instance->new_version, res_code, err); + goto out_error; + } + + instance->meta = cJSON_Parse(instance->estr.buff); + if(instance->meta == NULL) + { + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Parse meta failed, req_version=%lu, invalid json: %s", instance->new_version, instance->estr.buff); + goto out_error; + } + instance->statistic.field[DRS_FS_FILED_RES_META] += 1; + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, cur_version=%lu, newjson: %s", + instance->cur_version, instance->estr.buff); + + instance->param->cbs.version_start(instance, instance->meta, instance->param->cbs.userdata); + instance->array = cJSON_GetObjectItem(instance->meta, "configs"); + instance->array_size = cJSON_GetArraySize(instance->array); + assert(instance->array_size > 0); + + easy_string_destroy(&instance->estr); + instance->status = FETCH_STATUS_FILE; + doris_fetch_next_confile_meta(instance); + doris_http_fetch_confile(instance); + return; + +out_error: + instance->statistic.field[DRS_FS_FILED_RES_NOMETA] += 1; + doris_request_restart_timer(instance, instance->param->retry_interval); + easy_string_destroy(&instance->estr); + doris_confile_ctx_destry(&instance->ctx); +} + +static void doris_http_fetch_meta(struct doris_instance *instance) +{ + u_int64_t balance_seed; + struct doris_http_callback curlcbs; + char metauri[128]; + + balance_seed = (((u_int64_t)rand()) << 32) | rand(); + + memset(&curlcbs, 0, sizeof(struct doris_http_callback)); + curlcbs.header_cb = doris_http_meta_header_cb; + curlcbs.write_cb = doris_http_meta_body_cb; + curlcbs.transfer_done_cb = doris_http_meta_done_cb; + curlcbs.userp = instance; + + instance->array_index = 0; + instance->cur_httpins = instance->httpins_master; + instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed); + if(instance->ctx.httpctx==NULL && instance->httpins_backup1!=NULL) + { + instance->cur_httpins = instance->httpins_backup1; + instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed); + } + if(instance->ctx.httpctx==NULL && instance->httpins_backup2!=NULL) + { + instance->cur_httpins = instance->httpins_backup2; + instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed); + } + + if(instance->ctx.httpctx != NULL) + { + snprintf(metauri, 128, "configmeta?version=%lu&businessid=%u", instance->new_version, instance->param->args.businessid); + if(!doris_http_launch_get_request(instance->ctx.httpctx, metauri)) + { + instance->status = FETCH_STATUS_META; + instance->statistic.field[DRS_FS_FILED_REQ_META] += 1; + } + else + { + instance->statistic.field[DRS_FS_FILED_REQ_FAIL] += 1; + doris_confile_ctx_destry(&instance->ctx); + doris_request_restart_timer(instance, instance->param->retry_interval); + } + if(instance->cur_httpins == instance->httpins_backup1) instance->statistic.field[DRS_FS_FILED_BACKUP1_REQ] += 1; + else if(instance->cur_httpins == instance->httpins_backup2) instance->statistic.field[DRS_FS_FILED_BACKUP2_REQ] += 1; + } + else + { + instance->statistic.field[DRS_FS_FILED_REQ_FAIL] += 1; + doris_request_restart_timer(instance, instance->param->retry_interval); + MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Launch meta GET failed: no active host found,req_version=%lu", instance->new_version); + } +} + +static void instance_fetch_cfg_timer_cb(int fd, short kind, void *userp) +{ + struct doris_instance *instance = (struct doris_instance *)userp; + + switch(instance->status) + { + case FETCH_STATUS_IDLE: + case FETCH_STATUS_META: + doris_http_fetch_meta(instance); + break; + + case FETCH_STATUS_FILE: + doris_http_fetch_confile(instance); + break; + default: assert(0);break; + } +} + +static void doris_client_fs_output_timer_cb(int fd, short kind, void *userp) +{ + struct doris_parameter *param=(struct doris_parameter *)userp; + struct timeval tv; + + FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_MST_CNN_SRV], 0, FS_OP_SET, param->param_master->connected_hosts); + FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_MST_FAIL_SRV], 0, FS_OP_SET, param->param_master->failed_hosts); + if(param->param_backup1 != NULL) + { + FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_BCK1_CNN_SRV], 0, FS_OP_SET, param->param_backup1->connected_hosts); + FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_BCK1_FAIL_SRV], 0, FS_OP_SET, param->param_backup1->failed_hosts); + } + if(param->param_backup2 != NULL) + { + FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_BCK2_CNN_SRV], 0, FS_OP_SET, param->param_backup2->connected_hosts); + FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_BCK2_FAIL_SRV], 0, FS_OP_SET, param->param_backup2->failed_hosts); + } + FS_passive_output(param->fsstat_handle); + tv.tv_sec = param->fsstat_period; + tv.tv_usec = 0; + evtimer_add(¶m->fs_timer_output, &tv); +} + +static int doris_client_register_field_stat(struct doris_parameter *param, void *runtime_log, struct event_base *evbase) +{ + const char *field_names[FSSTAT_DORIS_FILED_NUM]={"ReqFail", "ReqMetas", "ResMetas", "ResNoNew", "ReqFiles", + "ResFiles", "ResFrags", "ResFragErr", "ResBytes", "ResVerErr", "ReqBackup1", "ReqBackup2"}; + const char *status_names[FSSTAT_DORIS_STATUS_NUM]={"MasSrvCned", "MasSrvFail", + "Bck1SrvCned", "Bck1SrvFail", "Bck2SrvCned", "Bck2SrvFail", "MemoryUsed", "HttpSession"}; + struct timeval tv; + int value; + + param->fsstat_handle = FS_create_handle(); + FS_set_para(param->fsstat_handle, OUTPUT_DEVICE, param->fsstat_filepath, strlen(param->fsstat_filepath)+1); + if(param->fsstat_print_mode == 1) + { + FS_set_para(param->fsstat_handle, PRINT_MODE, ¶m->fsstat_print_mode, sizeof(param->fsstat_print_mode)); + } + else + { + FS_set_para(param->fsstat_handle, PRINT_MODE, ¶m->fsstat_print_mode, sizeof(param->fsstat_print_mode)); + value = 1; + FS_set_para(param->fsstat_handle, FLUSH_BY_DATE, &value, sizeof(value)); + } + value = param->fsstat_period; + FS_set_para(param->fsstat_handle, STAT_CYCLE, &value, sizeof(value)); + value = 0; + FS_set_para(param->fsstat_handle, CREATE_THREAD, &value, sizeof(value)); + FS_set_para(param->fsstat_handle, APP_NAME, param->fsstat_appname, strlen(param->fsstat_appname)+1); + FS_set_para(param->fsstat_handle, STATS_SERVER_IP, param->fsstat_dst_ip, strlen(param->fsstat_dst_ip)+1); + FS_set_para(param->fsstat_handle, STATS_SERVER_PORT, ¶m->fsstat_dst_port, sizeof(param->fsstat_dst_port)); + + for(int i=0; ifsstat_field[i] = FS_register(param->fsstat_handle, FS_STYLE_FIELD, FS_CALC_CURRENT, field_names[i]); + } + for(int i=0; ifsstat_status[i] = FS_register(param->fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, status_names[i]); + } + FS_start(param->fsstat_handle); + + evtimer_assign(¶m->fs_timer_output, evbase, doris_client_fs_output_timer_cb, param); + tv.tv_sec = param->fsstat_period; + tv.tv_usec = 0; + evtimer_add(¶m->fs_timer_output, &tv); + return 0; +} + +struct doris_parameter *doris_parameter_new(const char *confile, struct event_base *manage_evbase, struct doris_callbacks *cbs, + struct doris_arguments *args, void *runtimelog) +{ + struct doris_parameter *param; + + param = (struct doris_parameter *)calloc(1, sizeof(struct doris_parameter)); + param->manage_evbase = manage_evbase; + param->cbs = *cbs; + param->args= *args; + + MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_fail_retry_interval", ¶m->retry_interval, 10); + MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_fragmet_size", ¶m->fetch_frag_size, 5242880); + MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_confile_max_tries", ¶m->fetch_max_tries, 3); + + MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_appname", param->fsstat_appname, 16, "DorisClient"); + MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_filepath", param->fsstat_filepath, 256, "./log/doris_client.fs"); + MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fsstat_log_interval", ¶m->fsstat_period, 10); + MESA_load_profile_int_def(confile, "DORIS_CLIENT", "fsstat_log_print_mode", ¶m->fsstat_print_mode, 1); + MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_dst_ip", param->fsstat_dst_ip, 64, "127.0.0.1"); + MESA_load_profile_int_def(confile, "DORIS_CLIENT", "fsstat_log_dst_port", ¶m->fsstat_dst_port, 8125); + + param->param_master = doris_http_parameter_new(confile, "DORIS_CLIENT.master_server", manage_evbase, runtimelog); + if(param->param_master == NULL) + { + return NULL; + } + if(doris_client_register_field_stat(param, runtimelog, manage_evbase)) + { + return NULL; + } + param->param_backup1 = doris_http_parameter_new(confile, "DORIS_CLIENT.backup1_server", manage_evbase, runtimelog); + param->param_backup2 = doris_http_parameter_new(confile, "DORIS_CLIENT.backup2_server", manage_evbase, runtimelog); + return param; +} + +static void doris_instance_statistic_timer_cb(int fd, short kind, void *userp) +{ + struct doris_instance *instance = (struct doris_instance *)userp; + struct timeval tv; + struct doris_statistics incr_statistic; + long long *plast_statistic = (long long*)&instance->statistic_last; + long long *pnow_statistic = (long long*)&instance->statistic; + long long *pinc_statistic = (long long*)&incr_statistic; + long long http_sessions=0; + + http_sessions += caculate_http_sessions_sum(instance->httpins_master); + http_sessions += caculate_http_sessions_sum(instance->httpins_backup1); + http_sessions += caculate_http_sessions_sum(instance->httpins_backup2); + instance->statistic.field[DRS_FS_STAT_HTTP_SESSIONS] = http_sessions; + + for(u_int32_t i=0; istatistic_last = instance->statistic; + + for(u_int32_t i=0; iparam->fsstat_handle, instance->param->fsstat_field[i], 0, FS_OP_ADD, incr_statistic.field[i]); + } + for(u_int32_t i=0; iparam->fsstat_handle, instance->param->fsstat_status[i], 0, FS_OP_ADD, incr_statistic.status[i]); + } + tv.tv_sec = instance->param->fsstat_period; + tv.tv_usec = 0; + event_add(&instance->timer_statistic, &tv); +} + +struct doris_instance *doris_instance_new(struct doris_parameter *param, struct event_base *worker_evbase, void *runtimelog) +{ + struct doris_instance *instance; + struct timeval tv; + + instance = (struct doris_instance *)calloc(1, sizeof(struct doris_instance)); + instance->param = param; + instance->worker_evbase = worker_evbase; + instance->runtime_log = runtimelog; + instance->cur_version = param->args.current_version; + instance->new_version = instance->cur_version + 1; //TODO + + instance->httpins_master = doris_http_instance_new(param->param_master, worker_evbase, runtimelog); + if(instance->httpins_master == NULL) + { + return NULL; + } + srand(time(NULL)); + + if(param->param_backup1 != NULL) + { + instance->httpins_backup1 = doris_http_instance_new(param->param_backup1, worker_evbase, runtimelog); + } + if(param->param_backup2 != NULL) + { + instance->httpins_backup2 = doris_http_instance_new(param->param_backup2, worker_evbase, runtimelog); + } + + evtimer_assign(&instance->timer_statistic, worker_evbase, doris_instance_statistic_timer_cb, instance); + tv.tv_sec = param->fsstat_period; + tv.tv_usec = 0; + evtimer_add(&instance->timer_statistic, &tv); + + evtimer_assign(&instance->timer_fetch, worker_evbase, instance_fetch_cfg_timer_cb, instance); + tv.tv_sec = 3; + tv.tv_usec = 0; + evtimer_add(&instance->timer_fetch, &tv); + return instance; +} + diff --git a/client/doris_client_fetch.h b/client/doris_client_fetch.h new file mode 100644 index 0000000..5b482eb --- /dev/null +++ b/client/doris_client_fetch.h @@ -0,0 +1,107 @@ +#ifndef __DORIS_CLIENT_FETCH_IN_H__ +#define __DORIS_CLIENT_FETCH_IN_H__ + +#include + +#include "doris_client.h" +#include "doris_client_http.h" + +struct easy_string +{ + char* buff; + size_t len; + size_t size; +}; + +enum FETCH_CFG_STATUS +{ + FETCH_STATUS_IDLE=0, + FETCH_STATUS_META, + FETCH_STATUS_FILE, +}; + +struct doris_parameter +{ + struct doris_callbacks cbs; + struct doris_arguments args; + + u_int32_t retry_interval; + u_int32_t fetch_frag_size; + u_int32_t fetch_max_tries; + + struct doris_http_parameter *param_master; + struct doris_http_parameter *param_backup1; + struct doris_http_parameter *param_backup2; + struct event_base *manage_evbase; + + screen_stat_handle_t fsstat_handle; + struct event fs_timer_output; + char fsstat_dst_ip[64]; + char fsstat_appname[16]; + char fsstat_filepath[256]; + u_int32_t fsstat_period; + int32_t fsstat_print_mode; + int32_t fsstat_dst_port; + int32_t fsstat_field[FSSTAT_DORIS_FILED_NUM]; + int32_t fsstat_status[FSSTAT_DORIS_STATUS_NUM]; +}; + +struct md5_long +{ + u_int64_t md5l; + u_int64_t md5h; +}; + +struct fetch_file_meta +{ + const char *table_name; + size_t size; + size_t curoffset; + u_int32_t cfg_num; + union { + char md5[16]; + struct md5_long md5long; + }; +}; + +struct doris_confile_ctx +{ + struct doris_http_ctx *httpctx; + + long res_code; + size_t contlength; + size_t contl_start; + size_t contl_end; + size_t contl_total; +}; + +struct doris_instance +{ + enum FETCH_CFG_STATUS status; + u_int32_t retry_times; + + struct doris_http_instance *cur_httpins; + int64_t cur_version; + int64_t new_version; + struct easy_string estr; + cJSON *meta, *array; + u_int32_t array_size; + u_int32_t array_index; + struct fetch_file_meta curmeta; + + struct doris_confile_ctx ctx; + + struct doris_http_instance *httpins_master; + struct doris_http_instance *httpins_backup1; + struct doris_http_instance *httpins_backup2; + + struct event_base *worker_evbase; + struct event timer_fetch; + struct doris_parameter *param; + struct event timer_statistic; + struct doris_statistics statistic, statistic_last; + void *runtime_log; +}; + +#endif + diff --git a/client/doris_client_http.cpp b/client/doris_client_http.cpp new file mode 100644 index 0000000..fbd9181 --- /dev/null +++ b/client/doris_client_http.cpp @@ -0,0 +1,374 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "doris_client_http.h" + +int32_t param_get_connected_hosts(struct doris_http_parameter *param) +{ + return param->connected_hosts; +} +int32_t param_get_failed_hosts(struct doris_http_parameter *param) +{ + return param->failed_hosts; +} + +static int _unfold_IP_range(char* ip_range, char***ip_list, int size) +{ + int i=0,count=0, ret=0; + int range_digits[5]; + memset(range_digits,0,sizeof(range_digits)); + ret=sscanf(ip_range,"%d.%d.%d.%d-%d",&range_digits[0],&range_digits[1],&range_digits[2],&range_digits[3],&range_digits[4]); + if(ret!=4&&ret!=5) + { + return 0; + } + if(ret==4&&range_digits[4]==0) + { + range_digits[4]=range_digits[3]; + } + for(i=0;i<5;i++) + { + if(range_digits[i]<0||range_digits[i]>255) + { + return 0; + } + } + count=range_digits[4]-range_digits[3]+1; + *ip_list=(char**)realloc(*ip_list, sizeof(char*)*(size+count)); + for(i=0;idstaddr_num = ipaddr_num; + dstaddr->dstaddrs = (u_int32_t *)calloc(1, sizeof(u_int32_t)*ipaddr_num); + for(i=0; idstaddrs[i]) != 1) + { + free(dstaddr->dstaddrs); + return -1; + } + free(balance_iplist[i]); + } + free(balance_iplist); + return 0; +} + +static int32_t load_and_init_server_group(char *dst_ipaddr, struct dst_ipaddr_group *dstaddrs, void *runtime_log) +{ + u_int32_t intval; + + intval = strlen(dst_ipaddr); + if(dst_ipaddr[intval-1] != ';') + { + dst_ipaddr[intval] = ';'; + dst_ipaddr[intval+1] = '\0'; + } + if(decode_one_specific_group_ip(dst_ipaddr, dstaddrs) < 0) + { + return -2; + } + return 0; +} + +static void conhash_delay_destroy_timer_cb(int fd, short kind, void *userp) +{ + struct time_event *delay_event=(struct time_event *)userp; + + conhash_instance_free((struct consistent_hash *)delay_event->data); + free(delay_event); +} + +static void load_balance_common_timer_start(struct event *time_event) +{ + struct timeval tv; + + tv.tv_sec = 2; + tv.tv_usec = 0; + evtimer_add(time_event, &tv); +} + +static void conhash_handle_delay_destroy(struct event_base *evbase, struct consistent_hash *conhash) +{ + struct time_event *delay_event; + + delay_event = (struct time_event *)malloc(sizeof(struct time_event)); + delay_event->data = conhash; + evtimer_assign(&delay_event->timer_event, evbase, conhash_delay_destroy_timer_cb, delay_event); + load_balance_common_timer_start(&delay_event->timer_event); +} + +static void conhash_insert_dest_host(struct dst_host_cnn_balance *balance) +{ + struct conhash_bucket bucket; + struct consistent_hash *tmphash, *newhash=NULL; + enum CONHASH_ERRCODE code; + + bucket.bucket_id = balance->dstip; + bucket.point_num = DEFAULT_HOST_CAPACITY * LOAD_BALANC_VIRT_TIMES;; + bucket.tag = NULL; + + newhash = conhash_instance_copy(balance->param->conhash); + code = conhash_insert_bucket(newhash, &bucket); + assert(code == CONHASH_OK); + + tmphash = balance->param->conhash; + balance->param->conhash = newhash; + conhash_handle_delay_destroy(balance->param->evbase, tmphash); +} + +static void conhash_remove_dest_host(struct dst_host_cnn_balance *balance) +{ + struct consistent_hash *tmphash, *newhash=NULL; + enum CONHASH_ERRCODE code; + + newhash = conhash_instance_copy(balance->param->conhash); + code = conhash_remove_bucket(newhash, balance->dstip, NULL); + assert(code == CONHASH_OK || code==CONHASH_BUCKET_NOT_FOUND); + + tmphash = balance->param->conhash; + balance->param->conhash = newhash; + conhash_handle_delay_destroy(balance->param->evbase, tmphash); +} + +static void client_bufferevent_error_cb(struct bufferevent *bev, short event, void *arg) +{ + struct dst_host_cnn_balance *balance = (struct dst_host_cnn_balance *)arg; + const char *errtype; + + if(event & BEV_EVENT_CONNECTED) + { + errtype = "connected"; + balance->connection_status = TCP_STATUS_CONNECTED; + balance->param->connected_hosts += 1; //仅有管理线程一个会进行链接探测 + if(balance->connect_failed) + { + balance->connect_failed = false; + balance->param->failed_hosts -= 1; + } + conhash_insert_dest_host(balance); + assert(balance->param->connected_hosts > 0); + assert(balance->param->failed_hosts >= 0); + } + else + { + if (event & BEV_EVENT_TIMEOUT) { + errtype = "Timed out"; + } + else if (event & BEV_EVENT_EOF) { + errtype = "disconnected"; + } + else if (event & BEV_EVENT_ERROR) { + errtype = "some other error"; + } + else { + errtype = "unkonwn error"; + } + bufferevent_free(bev); + balance->bev = NULL; + + if(balance->connection_status == TCP_STATUS_CONNECTED) + { + balance->param->connected_hosts -= 1; + } + if(!balance->connect_failed) + { + balance->connect_failed = true; + balance->param->failed_hosts += 1; + } + balance->connection_status = TCP_STATUS_DISCONNECT; + + load_balance_common_timer_start(&balance->timer_detect); + conhash_remove_dest_host(balance); + + assert(balance->param->connected_hosts >= 0); + assert(balance->param->failed_hosts > 0); + } + + MESA_HANDLE_RUNTIME_LOGV2(balance->param->runtime_log, RLOG_LV_INFO, "connection event: %s, addr: %s", errtype, balance->srvaddr); +} + +int do_bufferevent_connection(struct doris_http_parameter *param, struct dst_host_cnn_balance *balance, struct sockaddr *server_addr) +{ + if(balance->connection_status==TCP_STATUS_CONNECTING) + { + return 0; + } + + if(NULL == (balance->bev = bufferevent_socket_new(param->evbase, -1, BEV_OPT_CLOSE_ON_FREE))) + { + assert(0);return -1; + } + + if(bufferevent_socket_connect(balance->bev, server_addr, sizeof(struct sockaddr_in))) + { + bufferevent_free(balance->bev); + balance->bev = NULL; + MESA_HANDLE_RUNTIME_LOGV2(param->runtime_log, RLOG_LV_FATAL, "bufferevent_socket_connect error: %s", strerror(errno)); + assert(0);return -2; + } + balance->connection_status = TCP_STATUS_CONNECTING; + bufferevent_setcb(balance->bev, NULL, NULL, client_bufferevent_error_cb, balance); + bufferevent_setwatermark(balance->bev, EV_WRITE, 100*1024*1024UL, 0); + bufferevent_enable(balance->bev, EV_READ|EV_WRITE|EV_PERSIST); + return 0; +} + +void param_connection_detect_timer_cb(int fd, short kind, void *userp) +{ + struct dst_host_cnn_balance *balance = (struct dst_host_cnn_balance *)userp; + + if(do_bufferevent_connection(balance->param, balance, (struct sockaddr*)&balance->addr)) + { + assert(0); + } +} + +static int32_t doris_launch_group_connection(struct doris_http_parameter *param, struct event_base* evbase) +{ + char ipaddr[64]; + + param->balance = (struct dst_host_cnn_balance *)calloc(1, sizeof(struct dst_host_cnn_balance)*param->ipgroup.dstaddr_num); + + for(u_int32_t i=0; iipgroup.dstaddr_num; i++) //组内 + { + param->balance[i].dstip = param->ipgroup.dstaddrs[i]; + param->balance[i].addr.sin_family = AF_INET; + param->balance[i].addr.sin_port = htons(param->manage_port); + param->balance[i].addr.sin_addr.s_addr = param->ipgroup.dstaddrs[i]; + param->balance[i].connection_status = TCP_STATUS_IDLE; + param->balance[i].param = param; + + inet_ntop(AF_INET, ¶m->ipgroup.dstaddrs[i], ipaddr, 64); + snprintf(param->balance[i].srvaddr, 64, "%s:%u", ipaddr, param->server_port); + + evtimer_assign(¶m->balance[i].timer_detect, evbase, param_connection_detect_timer_cb, ¶m->balance[i]); + + if(do_bufferevent_connection(param, ¶m->balance[i], (struct sockaddr *)¶m->balance[i].addr)) + { + MESA_HANDLE_RUNTIME_LOGV2(param->runtime_log, RLOG_LV_FATAL, "do_bufferevent_connection error: %s.", strerror(errno)); + return -1; + } + } + return 0; +} + +struct doris_http_parameter *doris_http_parameter_new(const char* profile_path, const char* section, struct event_base* evbase, void *runtime_log) +{ + struct doris_http_parameter *param; + u_int32_t intval; + char dst_ipaddr[8192]; + + param = (struct doris_http_parameter *)calloc(1, sizeof(struct doris_http_parameter)); + param->runtime_log = runtime_log; + param->evbase = evbase; + + //multi curl + MESA_load_profile_uint_def(profile_path, section, "max_connection_per_host", &intval, 1); + param->maximum_host_cnns = intval; + MESA_load_profile_uint_def(profile_path, section, "max_cnnt_pipeline_num", &intval, 20); + param->maximum_pipelines = intval; + MESA_load_profile_uint_def(profile_path, section, "max_curl_transfer_timeout_s", &intval, 0); + param->transfer_timeout = intval; + MESA_load_profile_int_def(profile_path, section, "https_connection_on", ¶m->ssl_connection, 0); + MESA_load_profile_uint_def(profile_path, section, "max_curl_session_num", ¶m->max_http_sessions, 30); + + //Server + if(MESA_load_profile_uint_def(profile_path, section, "http_server_listen_port", ¶m->server_port, 9898) < 0) + { + free(param); + MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] http_server_listen_port not found.", profile_path, section); + return NULL; + } + MESA_load_profile_uint_def(profile_path, section, "http_server_manage_port", ¶m->manage_port, 2233); + + if(MESA_load_profile_string_nodef(profile_path, section, "http_server_ip_list", dst_ipaddr, 8192) < 0) + { + free(param); + MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] http_server_ip_list not found.", profile_path, section); + return NULL; + } + + if(load_and_init_server_group(dst_ipaddr, ¶m->ipgroup, runtime_log)) + { + MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Decode %s [%s] http_server_ip_list format error: %s", profile_path, section, dst_ipaddr); + assert(0);return NULL; + } + param->conhash = conhash_instance_new(NULL, 0);; + if(doris_launch_group_connection(param, evbase)) + { + assert(0);return NULL; + } + return param; +} + +struct doris_http_instance *doris_http_instance_new(struct doris_http_parameter *param, struct event_base* evbase, void *runtimelog) +{ + struct doris_http_instance *instance; + struct doris_curl_multihd *multihd; + + instance = (struct doris_http_instance *)calloc(1, sizeof(struct doris_http_instance)); + instance->runtime_log = runtimelog; + instance->evbase = evbase; + instance->param = param; + + //为每一个minio IP都创建一个multi_handle + instance->server_hosts = new map; + for(u_int32_t i=0; iipgroup.dstaddr_num; i++) + { + multihd = doris_initialize_multihd_for_host(instance, ¶m->balance[i]); + instance->server_hosts->insert(make_pair(param->ipgroup.dstaddrs[i], multihd)); + } + return instance; +} + diff --git a/client/doris_client_http.h b/client/doris_client_http.h new file mode 100644 index 0000000..6f829ab --- /dev/null +++ b/client/doris_client_http.h @@ -0,0 +1,108 @@ +#ifndef __DORIS_CLIENT_HTTP_IN_H__ +#define __DORIS_CLIENT_HTTP_IN_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "doris_client_transfer.h" +#include "nirvana_conhash.h" + +using namespace std; + +#ifndef __FILENAME__ +#define __FILENAME__ __FILE__ +#endif +#define MESA_HANDLE_RUNTIME_LOGV2(handle, lv, fmt, args...) \ + MESA_handle_runtime_log((handle), (lv), "DorisClient", "%s:%d, " fmt, __FILENAME__, __LINE__, ##args) + +#define DEFAULT_HOST_CAPACITY 4 +#define LOAD_BALANC_VIRT_TIMES 16 + +enum TCP_CONNECTION_STATUS +{ + TCP_STATUS_IDLE=0, + TCP_STATUS_CONNECTING, //只有正常发起链接才会缓存数据 + TCP_STATUS_CONNECTED, + TCP_STATUS_DISCONNECT, +}; + + +struct dst_ipaddr_group +{ + u_int32_t *dstaddrs; + u_int32_t dstaddr_num; +}; + +struct doris_http_parameter; +struct dst_host_cnn_balance +{ + struct sockaddr_in addr; + char srvaddr[64]; + + struct event timer_detect; + struct bufferevent *bev; + + u_int32_t dstip; + enum TCP_CONNECTION_STATUS connection_status; + bool connect_failed; + + struct doris_http_parameter *param; +}; + +struct doris_http_parameter +{ + u_int32_t server_port; + u_int32_t manage_port; + int32_t timer_period; + u_int32_t max_http_sessions; + int32_t connected_hosts; + int32_t failed_hosts; + int32_t ssl_connection; + + struct dst_ipaddr_group ipgroup; + struct consistent_hash *conhash; + struct dst_host_cnn_balance *balance; + + long maximum_host_cnns; + long transfer_timeout;//传输总时间限制 + long maximum_pipelines; + void *runtime_log; + struct event_base* evbase; +}; + +struct doris_http_instance +{ + struct event_base* evbase; + SSL_CTX *ssl_instance; + void *privdata; + + map *server_hosts; + + struct doris_http_parameter *param; + void *runtime_log; +}; + +struct time_event +{ + struct event timer_event; + void *data; +}; + +int32_t param_get_connected_hosts(struct doris_http_parameter *param); +int32_t param_get_failed_hosts(struct doris_http_parameter *param); + +struct doris_http_parameter *doris_http_parameter_new(const char* profile_path, const char* section, struct event_base* evbase, void *runtime_log); +struct doris_http_instance *doris_http_instance_new(struct doris_http_parameter *param, struct event_base* evbase, void *runtimelog); + +#endif + diff --git a/client/doris_client_transfer.cpp b/client/doris_client_transfer.cpp new file mode 100644 index 0000000..3850461 --- /dev/null +++ b/client/doris_client_transfer.cpp @@ -0,0 +1,379 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "doris_client_http.h" +#include "doris_client_transfer.h" +#include "nirvana_conhash.h" + +void doris_http_ctx_reset(struct doris_http_ctx *ctx, struct doris_http_callback *cb) +{ + struct doris_curl_multihd *multidata=ctx->multidata; + struct doris_http_instance *instance=ctx->instance; + + if(ctx->curl != NULL) + { + curl_multi_remove_handle(ctx->multidata->multi_hd, ctx->curl); + curl_easy_cleanup(ctx->curl); + ctx->curl = NULL; + } + if(ctx->headers != NULL) + { + curl_slist_free_all(ctx->headers); + } + memset(ctx, 0, sizeof(struct doris_http_ctx)); + ctx->multidata = multidata; + ctx->instance = instance; + ctx->cb = *cb; +} + +void doris_http_ctx_destroy(struct doris_http_ctx *ctx) +{ + if(ctx->curl != NULL) + { + curl_multi_remove_handle(ctx->multidata->multi_hd, ctx->curl); + curl_easy_cleanup(ctx->curl); + } + if(ctx->headers != NULL) + { + curl_slist_free_all(ctx->headers); + } + free(ctx); +} + +struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, struct doris_http_callback *cb, u_int64_t balance_seed) +{ + struct doris_http_ctx *ctx; + struct doris_curl_multihd *multidata; + struct conhash_bucket result; + + if(CONHASH_OK != conhash_lookup_bucket_int(instance->param->conhash, balance_seed, &result)) + { + return NULL; + } + assert(instance->server_hosts->find(result.bucket_id) != instance->server_hosts->end()); + multidata = instance->server_hosts->find(result.bucket_id)->second; + + ctx = (struct doris_http_ctx *)calloc(1, sizeof(struct doris_http_ctx)); + ctx->instance = instance; + ctx->multidata = multidata; + ctx->cb = *cb; + return ctx; +} + +long long caculate_http_sessions_sum(const struct doris_http_instance *instance) +{ + map::iterator iter; + long long sessions = 0; + + if(instance == NULL) return 0; + for(iter=instance->server_hosts->begin(); iter!=instance->server_hosts->end(); iter++) + { + sessions += iter->second->sessions; + } + return sessions; +} + + +static inline void curl_set_common_options(CURL *curl, long transfer_timeout, char *errorbuf) +{ + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 1000L); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //测试发现多链接有某链接接收卡住的情况 + //ctx->error="Operation too slow. Less than 100 bytes/sec transferred the last 10 seconds" + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 10L); + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "Doris Client Linux X64"); +} + +static size_t curl_response_write_cb(void *ptr, size_t size, size_t count, void *userp) +{ + struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp; + CURLcode code=CURLE_OK; + + if(ctx->res_code == 0) //首次应答时先看应答码是否是200 + { + code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code); + } + if(ctx->cb.write_cb != NULL) + { + ctx->cb.write_cb((const char*)ptr, size*count, code, ctx->res_code, ctx->cb.userp); + } + return size*count; +} + +static size_t curl_response_header_cb(void *ptr, size_t size, size_t count, void *userp) +{ + struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp; + size_t len=size*count; + CURLcode code=CURLE_OK; + + if(ctx->res_code == 0) //首次应答时先看应答码是否是200 + { + code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code); + } + if(ctx->cb.header_cb != NULL) + { + ctx->cb.header_cb((const char*)ptr, len, code, ctx->res_code, ctx->cb.userp); + } + return len; +} + +void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header) +{ + ctx->headers = curl_slist_append(ctx->headers, header); +} + +int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri) +{ + char minio_url[2048]; + + assert(ctx->curl == NULL); + if(NULL == (ctx->curl=curl_easy_init())) + { + return -1; + } + + if(ctx->instance->param->ssl_connection) + { + snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri); + curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L); + } + else + { + snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri); + } + curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); + + if(ctx->headers != NULL) //LOSF模式或者正常模式下按照Range获取 + { + curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); + } + curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb); + curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); + curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb); + curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); + curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); + curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); + + if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl)) + { + assert(0); + return -2; + } + ctx->transfering = 1; + return 0; +} + +int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, const char *data, size_t data_len) +{ + char minio_url[2048]; + + assert(ctx->curl == NULL); + if(NULL == (ctx->curl=curl_easy_init())) + { + return -1; + } + + doris_http_ctx_add_header(ctx, "Expect:"); //注意POST方法与Expect关系,要明确给出CURLOPT_POSTFIELDSIZE + if(ctx->instance->param->ssl_connection) + { + snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri); + curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L); + } + else + { + snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri); + } + curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); + curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); + curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, data_len); //填充Content-Length,在CURLOPT_COPYPOSTFIELDS之前设置 + if(data_len > 0) + { + curl_easy_setopt(ctx->curl, CURLOPT_COPYPOSTFIELDS, data); + } + curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); + curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb); + curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); + if(ctx->cb.header_cb != NULL) + { + curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb); + curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); + } + curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); + curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); + + if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl)) + { + assert(0); + return -2; + } + ctx->transfering = 1; + return 0; +} + +static void check_multi_info(CURLM *multi) +{ + CURLMsg *msg; + int msgs_left; + struct doris_http_ctx *ctx; + CURL *easy; + CURLcode res; + + while((msg = curl_multi_info_read(multi, &msgs_left))) + { + if(msg->msg != CURLMSG_DONE) + { + continue; + } + + easy = msg->easy_handle; + res = msg->data.result; + curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ctx); + curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &ctx->res_code); + curl_multi_remove_handle(multi, easy); + curl_easy_cleanup(easy); + ctx->curl = NULL; + ctx->transfering = 0; + ctx->res = res; + + ctx->cb.transfer_done_cb(ctx->res, ctx->res_code, ctx->error, ctx->cb.userp); + } +} + +/* Called by libevent when we get action on a multi socket */ +static void libevent_socket_event_cb(int fd, short action, void *userp) +{ + struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; //from event_assign + CURLMcode rc; + int what, still_running; + + what = ((action&EV_READ)?CURL_CSELECT_IN:0) | ((action & EV_WRITE)?CURL_CSELECT_OUT:0); + + rc = curl_multi_socket_action(multidata->multi_hd, fd, what, &still_running); + multidata->sessions = still_running; + assert(rc==CURLM_OK); + + check_multi_info(multidata->multi_hd); + if(still_running<=0 && evtimer_pending(&multidata->timer_event, NULL)) + { + evtimer_del(&multidata->timer_event); + } +} + +/* Called by libevent when our timeout expires */ +static void libevent_timer_event_cb(int fd, short kind, void *userp) +{ + struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; + CURLMcode rc; + int still_running; + + rc = curl_multi_socket_action(multidata->multi_hd, CURL_SOCKET_TIMEOUT, 0, &still_running); + multidata->sessions = still_running; + assert(rc==CURLM_OK); + check_multi_info(multidata->multi_hd); +} + +static int curl_socket_function_cb(CURL *curl, curl_socket_t sockfd, int what, void *userp, void *sockp) +{ + struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; + struct curl_socket_data *sockinfo = (struct curl_socket_data *)sockp; //curl_multi_assign, for socket + int action; + + if(what == CURL_POLL_REMOVE) + { + if(sockinfo != NULL) + { + event_del(&sockinfo->sock_event); + free(sockinfo); + } + } + else + { + if(sockinfo == NULL) + { + sockinfo = (struct curl_socket_data *)calloc(1, sizeof(struct curl_socket_data)); + curl_multi_assign(multidata->multi_hd, sockfd, sockinfo); + } + else + { + event_del(&sockinfo->sock_event); + } + + action = (what&CURL_POLL_IN?EV_READ:0)|(what&CURL_POLL_OUT?EV_WRITE:0)|EV_PERSIST; + event_assign(&sockinfo->sock_event, multidata->evbase, sockfd, action, libevent_socket_event_cb, multidata); + event_add(&sockinfo->sock_event, NULL); + } + + return 0; +} + +static int curl_timer_function_cb(CURLM *multi, long timeout_ms, void *userp) +{ + struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; + struct timeval timeout; + CURLMcode rc; + int still_running; + + timeout.tv_sec = timeout_ms/1000; + timeout.tv_usec = (timeout_ms%1000)*1000; + + if(timeout_ms == 0) + { + //timeout_ms is 0 means we should call curl_multi_socket_action/curl_multi_perform at once. + //To initiate the whole process(inform CURLMOPT_SOCKETFUNCTION callback) or when timeout occurs. + rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running); + multidata->sessions = still_running; + assert(rc==CURLM_OK); + } + else if(timeout_ms == -1) //timeout_ms is -1 means we should delete the timer. + { + //call curl_multi_socket_action to update multidata->sessions, otherwise it will not be updated to 0 + //when all transfers complete in some occasions(eg, GET, some objects hited while others miss). + rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running); + multidata->sessions = still_running; + evtimer_del(&multidata->timer_event); + } + else //update the timer to the new value. + { + evtimer_add(&multidata->timer_event, &timeout); + } + + return 0; //0-success; -1-error +} + + +//为每个Minio Host创建各自的curl multi handle +struct doris_curl_multihd *doris_initialize_multihd_for_host(struct doris_http_instance *instance, struct dst_host_cnn_balance *host) +{ + struct doris_curl_multihd *multidata; + + multidata = (struct doris_curl_multihd *)calloc(1, sizeof(struct doris_curl_multihd)); + multidata->multi_hd = curl_multi_init(); + multidata->evbase = instance->evbase; + multidata->host = host; + + curl_multi_setopt(multidata->multi_hd, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX); + curl_multi_setopt(multidata->multi_hd, CURLMOPT_MAX_HOST_CONNECTIONS, instance->param->maximum_host_cnns); + curl_multi_setopt(multidata->multi_hd, CURLMOPT_MAX_PIPELINE_LENGTH, instance->param->maximum_pipelines); + curl_multi_setopt(multidata->multi_hd, CURLMOPT_SOCKETFUNCTION, curl_socket_function_cb); + curl_multi_setopt(multidata->multi_hd, CURLMOPT_SOCKETDATA, multidata); //curl_socket_function_cb *userp + curl_multi_setopt(multidata->multi_hd, CURLMOPT_TIMERFUNCTION, curl_timer_function_cb); + curl_multi_setopt(multidata->multi_hd, CURLMOPT_TIMERDATA, multidata); + evtimer_assign(&multidata->timer_event, instance->evbase, libevent_timer_event_cb, multidata); + return multidata; +} + + diff --git a/client/doris_client_transfer.h b/client/doris_client_transfer.h new file mode 100644 index 0000000..6d1a9f5 --- /dev/null +++ b/client/doris_client_transfer.h @@ -0,0 +1,64 @@ +#ifndef __DORIS_HTTP_TRANSFER_H__ +#define __DORIS_HTTP_TRANSFER_H__ + +#include +#include + +#include + +struct doris_http_callback +{ + void *userp; + void (*header_cb)(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp); //HEAD/GET不可空 + void (*write_cb)(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp); //GET时不为空;其他为空时不回调 + void (*read_process_cb)(void *data, size_t data_offset, void *userp); //PUT时,为空时不回调 + void (*transfer_done_cb)(CURLcode res, long res_code, const char *err, void *userp); //不可空 +}; + +struct doris_http_ctx +{ + struct doris_curl_multihd *multidata; + CURL *curl; + struct curl_slist *headers; + char error[CURL_ERROR_SIZE]; + struct doris_http_callback cb; + CURLcode res; + long res_code; + int32_t transfering; + + char *put_data; + struct evbuffer *put_evbuf; + size_t put_length; + size_t put_offset; + struct doris_http_instance *instance; +}; + +struct dst_host_cnn_balance; +struct doris_curl_multihd +{ + struct dst_host_cnn_balance *host; + volatile long long sessions; //当前正在进行GET/PUT的会话数 + CURLM *multi_hd; + struct event timer_event; //curl动作的定时器 + struct event_base* evbase; +}; + +struct doris_curl_multihd *doris_initialize_multihd_for_host(struct doris_http_instance *instance, struct dst_host_cnn_balance *host); +struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, struct doris_http_callback *cb, u_int64_t balance_seed); +void doris_http_ctx_destroy(struct doris_http_ctx *ctx); +void doris_http_ctx_reset(struct doris_http_ctx *ctx, struct doris_http_callback *cb); + +void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header); + +int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri); +int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, const char *data, size_t data_len); +long long caculate_http_sessions_sum(const struct doris_http_instance *instance); + + +struct curl_socket_data +{ + struct event sock_event; +}; + +#endif + diff --git a/client/nirvana_conhash.cpp b/client/nirvana_conhash.cpp new file mode 100644 index 0000000..9a95403 --- /dev/null +++ b/client/nirvana_conhash.cpp @@ -0,0 +1,482 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "nirvana_murmurhash.h" +#include "nirvana_conhash.h" + +using namespace std; + +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +struct ch_point +{ + u_int32_t bucket_id; + u_int32_t bucket_index; /* which backend it belongs to, use IP address */ + u_int64_t hit_cnt; + u_int64_t point_val; /* hash code of this nodes, it is used to map node to consistent hash cycle */ +}; + +struct ch_bucket_inner +{ + struct conhash_bucket bucket; + + struct ch_point point_array[CONHASH_MAX_POINTS_PER_BUCKET]; //仅使用point_val成员去重 + int32_t is_valid; + int32_t bucket_index; + u_int64_t hit_cnt; +}; + +struct consistent_hash +{ + struct ch_bucket_inner *bucket_array; + u_int32_t bucket_array_size; + u_int32_t bucket_cnt; + u_int32_t point_num; + struct ch_point *point_array; + map *map_id_index; +}; + +//coefficient of variation of the RMSD +double conhash_calulate_CVRSMD(struct consistent_hash *ch) +{ + struct ch_bucket_inner* b=NULL; + u_int32_t i=0; + double sum_hit=0, sum_point=0; + double MSE=0,RMSD=0,CVRMSD=0; + for(i=0;ibucket_array_size;i++) + { + b=ch->bucket_array+i; + if(b->is_valid==0) + { + continue; + } + sum_hit+=(double)b->hit_cnt; + sum_point+=(double)b->bucket.point_num; + } + for(i=0;ibucket_array_size;i++) + { + b=ch->bucket_array+i; + if(b->is_valid==0) + { + continue; + } + MSE+=pow(b->hit_cnt-(b->bucket.point_num*sum_hit)/sum_point,2); + } + RMSD = sqrt(MSE/ch->bucket_cnt); + CVRMSD = RMSD/(sum_hit/ch->bucket_cnt); + return CVRMSD; +} + +static int qsort_cmp_by_key_increase(const void* a, const void* b) +{ + if(((const struct ch_point*)a)->point_val > ((const struct ch_point*)b)->point_val) + { + return 1; + } + else if(((const struct ch_point*)a)->point_val == ((const struct ch_point*)b)->point_val) + { + return 0; + } + else + { + return -1; + } +} + +static int qsort_cmp_by_key_decrease(const void* a, const void* b) +{ + if(((const struct ch_point*)a)->point_val > ((const struct ch_point*)b)->point_val) + { + return -1; + } + else if(((const struct ch_point*)a)->point_val == ((const struct ch_point*)b)->point_val) + { + return 0; + } + else + { + return 1; + } +} + +// (vector& nums, int target) +static u_int32_t search_up_bound(u_int64_t target, const void *base, + int32_t nmemb, size_t size,int val_offset) +{ + int32_t low = 0, high = nmemb-1, mid; + + // Invariant: the desired index is between [low, high+1] + + while (low <= high) + { + mid = low + (high-low)/2; + if(*(u_int64_t*)((char*)base+size*mid+val_offset) < target) + { + low = mid+1; + } + else + { + high = mid-1; + } + } + if(low == nmemb) + { + low=0; + } + // (1) At this point, low > high. That is, low >= high+1 + // (2) From the invariant, we know that the index is between [low, high+1], so low <= high+1. Follwing from (1), now we know low == high+1. + // (3) Following from (2), the index is between [low, high+1] = [low, low], which means that low is the desired index + // Therefore, we return low as the answer. You can also return high+1 as the result, since low == high+1 + return low; +} + +//保证相同的bucket_id&&point_index生成相同的point_id +static u_int64_t bucket_gen_uniq_point(struct ch_bucket_inner *inner_bucket, u_int32_t cur_point_index) +{ + u_int64_t x=0, seed; + u_int32_t hash, i=0; + + seed = (((u_int64_t)cur_point_index)<<32) | inner_bucket->bucket.bucket_id; + hash = murmurhash2(&seed, sizeof(u_int64_t), 23068673); + x = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id; + + while(i != cur_point_index) + { + for(i=0; ipoint_array[i].point_val) //冲突 + { + seed = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id; + hash = murmurhash2(&seed, sizeof(u_int64_t), 23068673); + x = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id; + i = 0; + break; + } + } + } + inner_bucket->point_array[cur_point_index].point_val = x; + return x; +} + +u_int32_t conhash_get_bucket_num(struct consistent_hash *ch) +{ + return ch->bucket_cnt; +} + +struct consistent_hash *conhash_instance_new(const struct conhash_bucket *buckets, uint32_t bucket_num) +{ + struct consistent_hash *ch=NULL; + u_int32_t i, j, k; + u_int64_t randval; + + for(i=0; i CONHASH_MAX_POINTS_PER_BUCKET) + { + return NULL; + } + } + ch = (struct consistent_hash *)calloc(1, sizeof(struct consistent_hash)); + + /*buckets*/ + ch->map_id_index = new map; + ch->bucket_array = (struct ch_bucket_inner*)calloc(1, sizeof(struct ch_bucket_inner)*bucket_num); + for(i=0; ibucket_array[i].bucket), &buckets[i], sizeof(struct conhash_bucket)); + ch->bucket_array[i].is_valid = 1; + ch->bucket_array[i].bucket_index = i; + ch->point_num += buckets[i].point_num; + ch->map_id_index->insert(make_pair(buckets[i].bucket_id, i)); + } + ch->bucket_cnt = bucket_num; + + /*global points*/ + ch->point_array = (struct ch_point*)calloc(1, sizeof(struct ch_point)*ch->point_num); + ch->bucket_array_size = bucket_num; + for(i=0, k=0; ibucket_array_size; i++) + { + for(j=0; jbucket_array[i].bucket.point_num; j++,k++) + { + randval = bucket_gen_uniq_point(&ch->bucket_array[i], j); + ch->point_array[k].bucket_id = ch->bucket_array[i].bucket.bucket_id; + ch->point_array[k].bucket_index = i; + ch->point_array[k].point_val = randval; + ch->point_array[k].hit_cnt = 0; + } + qsort(ch->bucket_array[i].point_array, ch->bucket_array[i].bucket.point_num, sizeof(struct ch_point), qsort_cmp_by_key_decrease); + } + qsort(ch->point_array, ch->point_num, sizeof(struct ch_point), qsort_cmp_by_key_increase); + return ch; +} + +void conhash_instance_free(struct consistent_hash *ch) +{ + free(ch->point_array); + free(ch->bucket_array); + delete ch->map_id_index; + free(ch); +} + +struct consistent_hash *conhash_instance_copy(struct consistent_hash *ch) +{ + struct consistent_hash *copy; + + copy = (struct consistent_hash *)calloc(1, sizeof(struct consistent_hash)); + copy->bucket_array_size = ch->bucket_array_size; + copy->bucket_cnt = ch->bucket_cnt; + copy->point_num = ch->point_num; + copy->bucket_array = (struct ch_bucket_inner*)calloc(sizeof(struct ch_bucket_inner), ch->bucket_array_size); + memcpy(copy->bucket_array, ch->bucket_array, sizeof(struct ch_bucket_inner)*ch->bucket_array_size); + + copy->point_array = (struct ch_point*)calloc(sizeof(struct ch_point), copy->point_num); + memcpy(copy->point_array, ch->point_array, sizeof(struct ch_point)*copy->point_num); + + copy->map_id_index = new map; + copy->map_id_index->insert(ch->map_id_index->begin(), ch->map_id_index->end()); + return copy; +} + +static enum CONHASH_ERRCODE conhash_add_points(struct consistent_hash *ch, struct ch_bucket_inner *inner_bucket, int32_t add_points) +{ + u_int64_t randval; + + if(inner_bucket->bucket.point_num + add_points > CONHASH_MAX_POINTS_PER_BUCKET) + { + assert(0);return CONHASH_ERR_INVALID_ARGS; + } + + ch->point_array = (struct ch_point *)realloc(ch->point_array,sizeof(struct ch_point)*(ch->point_num+add_points)); + + for(u_int32_t i=ch->point_num; ipoint_num+add_points; i++) + { + randval = bucket_gen_uniq_point(inner_bucket, inner_bucket->bucket.point_num); + inner_bucket->bucket.point_num++; + ch->point_array[i].bucket_id = inner_bucket->bucket.bucket_id; + ch->point_array[i].bucket_index = inner_bucket->bucket_index; + ch->point_array[i].point_val = randval; + ch->point_array[i].hit_cnt = 0; + } + ch->point_num += add_points; + qsort(inner_bucket->point_array, inner_bucket->bucket.point_num, sizeof(struct ch_point), qsort_cmp_by_key_decrease); + qsort(ch->point_array, ch->point_num, sizeof(struct ch_point), qsort_cmp_by_key_increase); + return CONHASH_OK; +} + +static enum CONHASH_ERRCODE conhash_del_points(struct consistent_hash *ch, struct ch_bucket_inner *inner_bucket, u_int32_t del_points) +{ + struct ch_point *tmp_points; + u_int32_t i, j, removed; + + if(inner_bucket->bucket.point_num == 0) + { + return CONHASH_OK; + } + if(inner_bucket->bucket.point_num < del_points) + { + assert(0);return CONHASH_ERR_INVALID_ARGS; + } + + tmp_points = (struct ch_point*)malloc(sizeof(struct ch_point)*ch->point_num); + memcpy(tmp_points, ch->point_array, sizeof(struct ch_point)*ch->point_num); + + for(i=0,j=0,removed=0; ipoint_num; i++) + { + if(removedbucket.bucket_id) + { + assert(inner_bucket->point_array[inner_bucket->bucket.point_num-1].point_val == tmp_points[i].point_val); + inner_bucket->bucket.point_num--; + removed++; + continue; + } + memcpy(&ch->point_array[j], &tmp_points[i], sizeof(struct ch_point)); + j++; + } + assert(removed == del_points); + free(tmp_points); + ch->point_num -= del_points; + //Sort is unnecessary after deletion. + return CONHASH_OK; +} + +enum CONHASH_ERRCODE conhash_insert_bucket(struct consistent_hash *ch, const struct conhash_bucket *bucket) +{ + struct ch_bucket_inner *inner_bucket=NULL; + u_int32_t i, bucket_index; + map::iterator iter; + enum CONHASH_ERRCODE code; + + if(bucket->point_num <= 0) + { + return CONHASH_ERR_INVALID_ARGS; + } + if((iter=ch->map_id_index->find(bucket->bucket_id)) != ch->map_id_index->end()) + { + return CONHASH_BUCKET_ALREADY_EXIST; + } + + if(ch->bucket_cnt < ch->bucket_array_size) + { + for(i=0; ibucket_array_size; i++) + { + if(ch->bucket_array[i].is_valid == 0) + { + bucket_index = i; + break; + } + } + assert(i < ch->bucket_array_size && ch->bucket_array[bucket_index].bucket.point_num==0); //一定能找到 + + inner_bucket = &ch->bucket_array[bucket_index]; + } + else + { + assert(ch->bucket_array_size == ch->bucket_cnt); + bucket_index = ch->bucket_cnt; + ch->bucket_array_size = ch->bucket_cnt + 1; + ch->bucket_array = (struct ch_bucket_inner*)realloc(ch->bucket_array, sizeof(struct ch_bucket_inner)*ch->bucket_array_size); + memset(&ch->bucket_array[bucket_index], 0, sizeof(struct ch_bucket_inner)); + inner_bucket = &ch->bucket_array[bucket_index]; + } + inner_bucket->bucket.bucket_id = bucket->bucket_id; + inner_bucket->bucket.tag = bucket->tag; + + if(CONHASH_OK != (code=conhash_add_points(ch, inner_bucket, bucket->point_num))) + { + return code; + } + inner_bucket->is_valid = 1; + inner_bucket->bucket_index = bucket_index; + inner_bucket->hit_cnt = 0; + ch->bucket_cnt++; + ch->map_id_index->insert(make_pair(bucket->bucket_id, bucket_index)); + return CONHASH_OK; +} + +enum CONHASH_ERRCODE conhash_remove_bucket(struct consistent_hash *ch, u_int32_t bucket_id, void (*free_cb)(void *tag, u_int32_t point_num)) +{ + struct ch_bucket_inner* inner_bucket=NULL; + u_int32_t bucket_index; + map::iterator iter; + enum CONHASH_ERRCODE code; + + if((iter=ch->map_id_index->find(bucket_id)) == ch->map_id_index->end()) + { + return CONHASH_BUCKET_NOT_FOUND; + } + bucket_index = iter->second; + assert(bucket_index < ch->bucket_array_size); + + inner_bucket = &ch->bucket_array[bucket_index]; + if(CONHASH_OK != (code=conhash_del_points(ch, inner_bucket, inner_bucket->bucket.point_num))) + { + return code; + } + ch->bucket_cnt--; + inner_bucket->is_valid = 0; + if(free_cb) + { + free_cb(inner_bucket->bucket.tag, inner_bucket->bucket.point_num); + } + inner_bucket->bucket.point_num = 0; + inner_bucket->bucket.tag = NULL; + ch->map_id_index->erase(bucket_id); + return CONHASH_OK; +} + +enum CONHASH_ERRCODE conhash_renew_bucket(struct consistent_hash *ch, struct conhash_bucket *bucket) +{ + struct ch_bucket_inner* inner_bucket=NULL; + u_int32_t bucket_index; + map::iterator iter; + + if((iter=ch->map_id_index->find(bucket->bucket_id)) == ch->map_id_index->end()) + { + assert(0);return CONHASH_BUCKET_NOT_FOUND; + } + bucket_index = iter->second; + assert(bucket_index < ch->bucket_array_size); + + inner_bucket = &ch->bucket_array[bucket_index]; + assert(inner_bucket->is_valid == 1); + inner_bucket->bucket.tag = bucket->tag; + + if(inner_bucket->bucket.point_num == bucket->point_num) + { + return CONHASH_OK; + } + else if(inner_bucket->bucket.point_num < bucket->point_num) + { + return conhash_add_points(ch, inner_bucket, bucket->point_num-inner_bucket->bucket.point_num); + } + else + { + return conhash_del_points(ch, inner_bucket, inner_bucket->bucket.point_num-bucket->point_num); + } +} + +enum CONHASH_ERRCODE conhash_lookup_bucket(struct consistent_hash *ch, const void* key, int len, struct conhash_bucket* result) +{ + int idx=0, bucket_index=0; + u_int64_t hash; + + if(ch->bucket_cnt == 0) + { + return CONHASH_NO_VALID_BUCKETS; + } + + hash = MurmurHash64A(key, len, 23068673); + idx = search_up_bound(hash, ch->point_array, ch->point_num, sizeof(struct ch_point), offsetof(struct ch_point, point_val)); + ch->point_array[idx].hit_cnt++; + bucket_index = ch->point_array[idx].bucket_index; + assert(ch->bucket_array[bucket_index].is_valid == 1); + ch->bucket_array[bucket_index].hit_cnt++; + memcpy(result, &(ch->bucket_array[bucket_index].bucket), sizeof(struct conhash_bucket)); + return CONHASH_OK; +} + +enum CONHASH_ERRCODE conhash_lookup_bucket_int(struct consistent_hash *ch, u_int64_t randint, struct conhash_bucket* result) +{ + int idx=0, bucket_index=0; + + if(ch->bucket_cnt == 0) + { + return CONHASH_NO_VALID_BUCKETS; + } + + idx = search_up_bound(randint, ch->point_array, ch->point_num, sizeof(struct ch_point), offsetof(struct ch_point, point_val)); + ch->point_array[idx].hit_cnt++; + bucket_index = ch->point_array[idx].bucket_index; + assert(ch->bucket_array[bucket_index].is_valid == 1); + ch->bucket_array[bucket_index].hit_cnt++; + memcpy(result, &(ch->bucket_array[bucket_index].bucket), sizeof(struct conhash_bucket)); + return CONHASH_OK; +} + +void conhash_dump_detail(struct consistent_hash *ch) +{ + /*for(u_int32_t i=0; ipoint_num; i++) + { + printf("bucket_id: %10u, bucket_index: %5u, point_val:%lx, hit_cnt: %lu\n", ch->point_array[i].bucket_id, + ch->point_array[i].bucket_index, ch->point_array[i].point_val, ch->point_array[i].hit_cnt); + } + + printf("\n\n\n\n");*/ + for(u_int32_t i=0; ibucket_cnt; i++) + { + if(ch->bucket_array[i].is_valid) + { + printf("bucket_id: %10u, bucket_index: %5u, hit_cnt: %lu\n", ch->bucket_array[i].bucket.bucket_id, + i, ch->bucket_array[i].hit_cnt); + } + } +} + diff --git a/client/nirvana_conhash.h b/client/nirvana_conhash.h new file mode 100644 index 0000000..42320b1 --- /dev/null +++ b/client/nirvana_conhash.h @@ -0,0 +1,50 @@ +#ifndef __NVN_CONSISTENT_HASH_H__ +#define __NVN_CONSISTENT_HASH_H__ + +#include + +#ifndef CONHASH_MAX_POINTS_PER_BUCKET +#define CONHASH_MAX_POINTS_PER_BUCKET 128 +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + +enum CONHASH_ERRCODE +{ + CONHASH_OK = 0, + CONHASH_ERR_INVALID_ARGS = -1, + CONHASH_BUCKET_NOT_FOUND = -2, + CONHASH_BUCKET_ALREADY_EXIST=-3, + CONHASH_NO_VALID_BUCKETS=-4, +}; + +struct conhash_bucket +{ + uint32_t bucket_id; + uint32_t point_num; /*should be not more than CONHASH_MAX_POINTS_PER_BUCKET*/ + void* tag; +}; +struct consistent_hash; + +/*API线程不安全*/ +struct consistent_hash *conhash_instance_new(const struct conhash_bucket *buckets, uint32_t bucket_num); +void conhash_instance_free(struct consistent_hash *ch); +struct consistent_hash *conhash_instance_copy(struct consistent_hash *ch); + +enum CONHASH_ERRCODE conhash_insert_bucket(struct consistent_hash *ch,const struct conhash_bucket* bucket); +enum CONHASH_ERRCODE conhash_remove_bucket(struct consistent_hash *ch, u_int32_t bucket_id, void (*free_cb)(void *tag, u_int32_t point_num)); +enum CONHASH_ERRCODE conhash_renew_bucket(struct consistent_hash *ch, struct conhash_bucket* bucket); /*更新point_num和tag*/ +enum CONHASH_ERRCODE conhash_lookup_bucket(struct consistent_hash *ch, const void* key, int len, struct conhash_bucket *result/*OUT*/); +enum CONHASH_ERRCODE conhash_lookup_bucket_int(struct consistent_hash *ch, u_int64_t randint, struct conhash_bucket* result); + +double conhash_calulate_CVRSMD(struct consistent_hash *p); +u_int32_t conhash_get_bucket_num(struct consistent_hash *ch); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/client/nirvana_murmurhash.cpp b/client/nirvana_murmurhash.cpp new file mode 100644 index 0000000..193bde9 --- /dev/null +++ b/client/nirvana_murmurhash.cpp @@ -0,0 +1,99 @@ +#include +#include +#include + +#include "nirvana_murmurhash.h" + +unsigned int murmurhash2(const void * key, int len, const unsigned int seed) +{ + // 'm' and 'r' are mixing constants generated offline. + // They're not really 'magic', they just happen to work well. + + const unsigned int m = 0x5bd1e995; + const int r = 24; + + // Initialize the hash to a 'random' value + + unsigned int h = seed ^ len; + + // Mix 4 bytes at a time into the hash + + const unsigned char * data = (const unsigned char *)key; + + while(len >= 4) + { + unsigned int k = *(unsigned int *)data; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + // Handle the last few bytes of the input array + + switch(len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + default:break; + }; + + // Do a few final mixes of the hash to ensure the last few + // bytes are well-incorporated. + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + return h; +} + +/*64-bit hash for 64-bit platforms*/ +u_int64_t MurmurHash64A(const void * key, int len, unsigned int seed) +{ + const uint64_t m = 0xc6a4a7935bd1e995; + const int r = 47; + uint64_t k, h = seed ^ (len * m); + const uint64_t * data = (const uint64_t *)key; + const uint64_t * end = data + (len/8); + + while (data != end) + { + k = *data++; + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + const unsigned char * data2 = (const unsigned char*)data; + + switch (len & 7) + { + case 7: h ^= uint64_t(data2[6]) << 48; + case 6: h ^= uint64_t(data2[5]) << 40; + case 5: h ^= uint64_t(data2[4]) << 32; + case 4: h ^= uint64_t(data2[3]) << 24; + case 3: h ^= uint64_t(data2[2]) << 16; + case 2: h ^= uint64_t(data2[1]) << 8; + case 1: h ^= uint64_t(data2[0]); + h *= m; + }; + + h ^= h >> r; + h *= m; + h ^= h >> r; + + return h; +} + + diff --git a/client/nirvana_murmurhash.h b/client/nirvana_murmurhash.h new file mode 100644 index 0000000..1467943 --- /dev/null +++ b/client/nirvana_murmurhash.h @@ -0,0 +1,16 @@ +#ifndef __NVN_MURMURHASH_H__ +#define __NVN_MURMURHASH_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +unsigned int murmurhash2(const void * key, int len, const unsigned int seed); +u_int64_t MurmurHash64A(const void * key, int len, unsigned int seed); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/include/doris_client.h b/include/doris_client.h new file mode 100644 index 0000000..fea6826 --- /dev/null +++ b/include/doris_client.h @@ -0,0 +1,70 @@ +#ifndef __DORIS_CLIENT_H__ +#define __DORIS_CLIENT_H__ + +#include +#include + +enum FSSTAT_DORIS_FILED_ITEMS +{ + DRS_FS_FILED_REQ_FAIL=0, + DRS_FS_FILED_REQ_META, + DRS_FS_FILED_RES_META, + DRS_FS_FILED_RES_NOMETA, + DRS_FS_FILED_REQ_FILES, + DRS_FS_FILED_RES_FILES, + DRS_FS_FILED_RES_FRAGS, + DRS_FS_FILED_RES_FRAGERR, + DRS_FS_FILED_RES_BYTES, + DRS_FS_FILED_RES_VERERR, + + DRS_FS_FILED_BACKUP1_REQ, + DRS_FS_FILED_BACKUP2_REQ, + + FSSTAT_DORIS_FILED_NUM, +}; + +enum FSSTAT_DORIS_STATUS_ITEMS +{ + DRS_FS_STAT_MST_CNN_SRV=0, + DRS_FS_STAT_MST_FAIL_SRV, + DRS_FS_STAT_BCK1_CNN_SRV, + DRS_FS_STAT_BCK1_FAIL_SRV, + DRS_FS_STAT_BCK2_CNN_SRV, + DRS_FS_STAT_BCK2_FAIL_SRV, + DRS_FS_STAT_MEMORY_USED, + DRS_FS_STAT_HTTP_SESSIONS, + + FSSTAT_DORIS_STATUS_NUM, +}; + +struct doris_statistics +{ + long long field[FSSTAT_DORIS_FILED_NUM]; + long long status[FSSTAT_DORIS_STATUS_NUM]; +}; + +struct doris_arguments +{ + int64_t current_version; //当前已获取完毕的最新版本号,将从它下一个版本取配置 + int32_t businessid; + int32_t judian_id; +}; + +struct doris_instance; +struct doris_callbacks +{ + void *userdata; + void (*version_start)(struct doris_instance *instance, cJSON *meta, void *userdata); //meta在整个版本声明周期内都有效 + void (*cfgfile_start)(struct doris_instance *instance, const char *tablename, size_t size, u_int32_t cfgnum, void *userdata); + void (*cfgfile_update)(struct doris_instance *instance, const char *data, size_t len, void *userdata); + void (*cfgfile_finish)(struct doris_instance *instance, void *userdata); + void (*version_error)(struct doris_instance *instance, void *userdata); //下载文件失败,该版本需要回滚 + void (*version_finish)(struct doris_instance *instance, void *userdata); +}; + +struct doris_parameter *doris_parameter_new(const char *confile, struct event_base *manage_evbase, struct doris_callbacks *cbs, + struct doris_arguments *args, void *runtimelog); +struct doris_instance *doris_instance_new(struct doris_parameter *param, struct event_base *worker_evbase, void *runtimelog); + +#endif + diff --git a/monitor/prometheus.yml b/monitor/prometheus.yml new file mode 100644 index 0000000..d81c6ed --- /dev/null +++ b/monitor/prometheus.yml @@ -0,0 +1,28 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + - job_name: "doris" + static_configs: + - targets: ['192.168.10.8:2233', '192.168.10.9:2233'] + diff --git a/monitor/閰嶇疆鍒嗗彂缃戠粶杩愯鐘舵-1626422695467.json b/monitor/閰嶇疆鍒嗗彂缃戠粶杩愯鐘舵-1626422695467.json new file mode 100644 index 0000000..8cfb5c9 --- /dev/null +++ b/monitor/閰嶇疆鍒嗗彂缃戠粶杩愯鐘舵-1626422695467.json @@ -0,0 +1,518 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHUES-8", + "label": "promethues-8", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.4" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "${DS_PROMETHUES-8}", + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "links": [], + "pageSize": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "State", + "colorMode": "cell", + "colors": [ + "#bf1b00", + "rgb(22, 191, 17)", + "#bf1b00" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "Value", + "preserveFormat": false, + "sanitize": false, + "thresholds": [ + "1" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "up", + "value": "1" + }, + { + "text": "down", + "value": "0" + } + ] + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "process", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "minio", + "value": "minio_filelog_astana" + } + ] + }, + { + "alias": "State", + "colorMode": "cell", + "colors": [ + "#bf1b00", + "rgb(13, 231, 28)", + "#bf1b00" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [ + "1" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "up", + "value": "1" + }, + { + "text": "down", + "value": "0" + } + ] + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "CPU", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Openfds", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "MAXfds", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "none" + }, + { + "alias": "MEM", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Threads", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Restart24H", + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Value #H", + "thresholds": [ + "1", + "4" + ], + "type": "number", + "unit": "none" + }, + { + "alias": "UserState", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#3f6833", + "#e24d42" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #I", + "thresholds": [ + "1", + "3" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "Normal", + "value": "1" + }, + { + "text": "down", + "value": "0" + }, + { + "text": "plug HTTP down", + "value": "10" + } + ] + }, + { + "alias": "MainVersion", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #J", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "LatestVersion", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #K", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "TotalCfgNum", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #L", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "up{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + }, + { + "expr": "0+process_main_version{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "J" + }, + { + "expr": "0+process_uptime_seconds{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "B" + }, + { + "expr": "0+sum_over_time(restart_times{job=~\"doris\"}[24h])", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "H" + }, + { + "expr": "irate(process_cpu_seconds_total{job=~\"doris\"}[1m])", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "C" + }, + { + "expr": "0+(process_resident_memory_bytes{job=~\"doris\"}/process_total_memory_bytes{job=~\"doris\"})", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "F" + }, + { + "expr": "0+process_active_threads{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "G" + }, + { + "expr": "0+process_max_fds{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "E" + }, + { + "expr": "0+process_open_fds{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "D" + }, + { + "expr": "0+user_defined_status{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "I" + }, + { + "expr": "0+latest_cfg_version{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "K" + }, + { + "expr": "0+total_config_num{job=~\"doris\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "L" + } + ], + "title": "Doris杩愯鐘舵", + "transform": "table", + "type": "table" + } + ], + "refresh": "10s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "閰嶇疆鍒嗗彂缃戠粶杩愯鐘舵", + "uid": "HZfW8wi7k", + "version": 5 +} \ No newline at end of file diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt new file mode 100644 index 0000000..3f609ed --- /dev/null +++ b/server/CMakeLists.txt @@ -0,0 +1,14 @@ +set (NIRVANA_PLATFORM_SRC doris_server_scandir.cpp doris_server_receive.cpp doris_server_http.cpp doris_server_main.cpp) + +add_definitions(-fPIC -Wall -g) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_CURRENT_SOURCE_DIR}/,,$(abspath $<))\"'") + +add_executable(doris ${NIRVANA_PLATFORM_SRC}) +target_link_libraries(doris doris_client_static libMesaMonitor libevent-static libevent-pthreads-static libcurl-static libevent-openssl-static openssl-ssl-static openssl-crypto-static cjson libLevelDB) +target_link_libraries(doris MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat2 pthread z dl) +set_target_properties(doris PROPERTIES CLEAN_DIRECT_OUTPUT 1) +target_include_directories(doris PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) +set_property(TARGET doris PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/include) + +INSTALL (DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bin/ DESTINATION bin) +INSTALL (TARGETS doris DESTINATION bin) diff --git a/server/bin/conf/doris_main.conf b/server/bin/conf/doris_main.conf new file mode 100644 index 0000000..5ad1007 --- /dev/null +++ b/server/bin/conf/doris_main.conf @@ -0,0 +1,57 @@ +[DORIS_SERVER] +worker_thread_num=2 +server_listen_port=9898 +manage_listen_port=2233 + +receive_config_way=2 +cache_file_frag_size=67108864 +store_config_path=./doris_store_path +receive_config_path_full=./doris_receive_path/full/index +receive_config_path_inc=./doris_receive_path/inc/index + +run_log_dir=./log +run_log_lv=20 +fsstat_log_appname=DorisServer +fsstat_log_filepath=./log/doris_server.fs +fsstat_log_interval=10 +fsstat_log_print_mode=1 +fsstat_log_dst_ip=192.168.10.90 +fsstat_log_dst_port=8125 + + + +[DORIS_CLIENT] +fetch_fail_retry_interval=5 +fetch_fragmet_size=5242880 +fetch_confile_max_tries=3 + +fsstat_log_appname=DorisClient +fsstat_log_filepath=./log/doris_client.fs +fsstat_log_interval=2 +fsstat_log_print_mode=1 +fsstat_log_dst_ip=192.168.10.90 +fsstat_log_dst_port=8125 + +[DORIS_CLIENT.master_server] +max_connection_per_host=1 +max_cnnt_pipeline_num=10 +https_connection_on=0 +max_curl_session_num=10 + +http_server_listen_port=9897 +http_server_manage_port=9897 +http_server_ip_list=192.168.10.8 + +[DORIS_CLIENT.backup1_server] +max_connection_per_host=1 +max_cnnt_pipeline_num=10 +https_connection_on=0 +max_curl_session_num=10 + +http_server_listen_port=9897 +http_server_manage_port=9897 +http_server_ip_list=192.168.11.241 + +[DORIS_CLIENT.backup2_server] + + diff --git a/server/bin/doris_dmn.sh b/server/bin/doris_dmn.sh new file mode 100644 index 0000000..5deddff --- /dev/null +++ b/server/bin/doris_dmn.sh @@ -0,0 +1,18 @@ +#!/bin/bash +while [ 1 ]; +do + count=`ls -l core.* |wc -l` + echo $count + if [ $count -lt 3 ] + then + echo "set unlimited" + ulimit -c unlimited + else + ulimit -c 0 + fi + + ./doris >> /dev/null + echo program crashed, restart at `date +"%w %Y/%m/%d, %H:%M:%S"` >> RESTART.log + sleep 30 +done + diff --git a/server/bin/doris_kill_by_mem.sh b/server/bin/doris_kill_by_mem.sh new file mode 100644 index 0000000..b25370d --- /dev/null +++ b/server/bin/doris_kill_by_mem.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +PROG="doris" + +while true; do + mem_used=`ps -C $PROG u | grep -v RSS | awk '{print $6}'` + mem_rate=`ps -C $PROG -o %mem | tail -1` + #cpu_used=`ps -C $PROG -o %cpu | tail -1` + #echo "`date` -- $PROG used mem : $mem_used, $mem_rate%, $cpu_used%" >> doris_kill_by_mem.log + if [ -z $mem_used ]; then + mem_used=0 + elif [ $mem_used -gt 102400000 ]; then + echo "`date` -- $PROG used mem : $mem_used, $mem_rate%, kill it!" >> RESTART.log + killall $PROG; + fi + + sleep 10 + +done diff --git a/server/bin/doris_start.sh b/server/bin/doris_start.sh new file mode 100644 index 0000000..850c260 --- /dev/null +++ b/server/bin/doris_start.sh @@ -0,0 +1,6 @@ +#!/bin/bash +source /etc/profile +killall doris_kill_by_mem.sh doris_dmn.sh doris +./doris_dmn.sh &>/dev/null & +./doris_kill_by_mem.sh &>/dev/null & + diff --git a/server/bin/doris_stop.sh b/server/bin/doris_stop.sh new file mode 100644 index 0000000..472da9f --- /dev/null +++ b/server/bin/doris_stop.sh @@ -0,0 +1,2 @@ +#!/bin/bash +killall doris_kill_by_mem.sh doris_dmn.sh doris diff --git a/server/bin/memchk.sh b/server/bin/memchk.sh new file mode 100644 index 0000000..121b855 --- /dev/null +++ b/server/bin/memchk.sh @@ -0,0 +1,2 @@ +#!/bin/sh +valgrind --tool=memcheck --leak-check=full --leak-resolution=high --error-limit=no --undef-value-errors=yes --log-file=valgrind.log ./doris diff --git a/server/doris_server_http.cpp b/server/doris_server_http.cpp new file mode 100644 index 0000000..69bbe29 --- /dev/null +++ b/server/doris_server_http.cpp @@ -0,0 +1,277 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "doris_server_main.h" +#include "doris_server_http.h" + + +extern struct nirvana_global_info g_doris_server_info; + +static inline void set_sockopt_keepalive(int sd, int keepidle, int keepintvl, int keepcnt) +{ + int keepalive = 1; + setsockopt(sd, SOL_SOCKET, SO_KEEPALIVE, (void*)&keepalive, sizeof(keepalive)); + setsockopt(sd, SOL_TCP, TCP_KEEPIDLE, (void*)&keepidle, sizeof(keepidle)); + setsockopt(sd, SOL_TCP, TCP_KEEPINTVL, (void*)&keepintvl, sizeof(keepintvl)); + setsockopt(sd, SOL_TCP, TCP_KEEPCNT, (void*)&keepcnt, sizeof(keepcnt)); +} + +static inline void set_listen_sockopt(int sd) +{ + if(g_doris_server_info.sock_recv_bufsize > 0) + { + setsockopt(sd, SOL_SOCKET, SO_RCVBUF, &g_doris_server_info.sock_recv_bufsize, sizeof(u_int32_t)); + } +} + +int doris_create_listen_socket(int bind_port) +{ + evutil_socket_t listener; + struct sockaddr_in sin; + + listener = socket(AF_INET, SOCK_STREAM, 0); + if(listener < 0) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "create socket error!\n"); + return -1; + } + set_sockopt_keepalive(listener, 300, 10, 2); + set_listen_sockopt(listener); + evutil_make_listen_socket_reuseable(listener); + + sin.sin_family = AF_INET; + sin.sin_addr.s_addr=htonl(INADDR_ANY); + sin.sin_port = htons(bind_port); + + if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0) + { + printf("bind socket to port: %d error: %s!\n", bind_port, strerror(errno)); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "bind socket to port: %d error: %s!\n", bind_port, strerror(errno)); + assert(0);return -2; + } + if (listen(listener, 1024)<0) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "listen socket 1024 error!\n"); + return -3; + } + evutil_make_socket_nonblocking(listener); + return listener; +} + +void doris_http_server_meta_cb(struct evhttp_request *req, void *arg) +{ + struct worker_statistic_info *statistic=(struct worker_statistic_info *)arg; + struct evkeyvalq params; + const char *version; + int64_t verlong; + char *endptr=NULL, length[64]; + struct version_list_node *vernode; + struct evbuffer *evbuf; + + statistic->statistic.field[DRS_FSSTAT_CLIENT_META_REQ] += 1; + if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) + { + statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1; + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid"); + return; + } + if(NULL == (version = evhttp_find_header(¶ms, "version"))) + { + statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1; + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no version found"); + return; + } + if(0==(verlong = strtol(version, &endptr, 10)) || *endptr!='\0') + { + statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1; + evhttp_send_error(req, HTTP_BADREQUEST, "Parameter version invalid"); + return; + } + + pthread_rwlock_rdlock(&g_doris_server_info.rwlock); + if(verlong > g_doris_server_info.cfgver_head->latest_version) + { + pthread_rwlock_unlock(&g_doris_server_info.rwlock); + statistic->statistic.field[DRS_FSSTAT_SEND_META_NONEW] += 1; + evhttp_send_error(req, HTTP_NOTMODIFIED, "No new configs found"); + return; + } + vernode = TAILQ_FIRST(&g_doris_server_info.cfgver_head->version_head); + while(vernode->version < verlong) + { + vernode = TAILQ_NEXT(vernode, version_node); + } + evbuf = evbuffer_new(); + evbuffer_add(evbuf, vernode->metacont, vernode->metalen); + sprintf(length, "%u", vernode->metalen); + pthread_rwlock_unlock(&g_doris_server_info.rwlock); + + statistic->statistic.field[DRS_FSSTAT_SEND_META_RES] += 1; + evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "application/json"); + evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "keep-alive"); + evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Length", length); + evhttp_send_reply(req, HTTP_OK, "OK", evbuf); + evbuffer_free(evbuf); +} + +void doris_response_file_range(struct evhttp_request *req, const char *tablename, + int64_t verlong, size_t start, size_t end, bool range, struct worker_statistic_info *statistic) +{ + struct version_list_node *vernode; + struct table_list_node *tablenode; + struct cont_frag_node *fragnode; + struct evbuffer *evbuf; + char length[128]; + size_t filesize, res_length=0, copy_len, offset=start; + + pthread_rwlock_rdlock(&g_doris_server_info.rwlock); + if(verlong > g_doris_server_info.cfgver_head->latest_version) + { + pthread_rwlock_unlock(&g_doris_server_info.rwlock); + statistic->statistic.field[DRS_FSSTAT_SEND_FILE_RES_404] += 1; + evhttp_send_error(req, HTTP_NOTFOUND, "Version too old"); + return; + } + vernode = TAILQ_FIRST(&g_doris_server_info.cfgver_head->version_head); + while(vernode->version < verlong) + { + vernode = TAILQ_NEXT(vernode, version_node); + } + tablenode = TAILQ_FIRST(&vernode->table_head); + while(tablenode!=NULL && strcmp(tablename, tablenode->tablename)) + { + tablenode = TAILQ_NEXT(tablenode, table_node); + } + if(tablenode==NULL || start>tablenode->filesize) + { + pthread_rwlock_unlock(&g_doris_server_info.rwlock); + statistic->statistic.field[DRS_FSSTAT_SEND_FILE_RES_404] += 1; + evhttp_send_error(req, HTTP_NOTFOUND, "No valid content found"); + return; + } + filesize = tablenode->filesize; + if(end==0 || end >= tablenode->filesize) + { + end = tablenode->filesize - 1; + } + evbuf = evbuffer_new(); + for(fragnode=TAILQ_FIRST(&tablenode->frag_head); fragnode!=NULL && fragnode->start<=end; fragnode=TAILQ_NEXT(fragnode, frag_node)) + { + if(offset > fragnode->end) + { + continue; + } + copy_len = (end>fragnode->end)?(fragnode->end-offset + 1):(end-offset + 1); + evbuffer_add(evbuf, fragnode->content+(offset-fragnode->start), copy_len); + offset += copy_len; + res_length += copy_len; + } + pthread_rwlock_unlock(&g_doris_server_info.rwlock); + + assert(res_length == end + 1 - start); + sprintf(length, "%lu", res_length); + statistic->statistic.field[DRS_FSSTAT_SEND_FILE_RES] += 1; + statistic->statistic.field[DRS_FSSTAT_SEND_FILE_BYTES] += res_length; + evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Length", length); + if(range) + { + sprintf(length, "bytes %lu-%lu/%lu", start, end, filesize); + evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Range", length); + } + evhttp_add_header(evhttp_request_get_output_headers(req), "Content-Type", "application/stream"); + evhttp_add_header(evhttp_request_get_output_headers(req), "Connection", "keep-alive"); + evhttp_send_reply(req, HTTP_OK, "OK", evbuf); +} + +void doris_http_server_file_cb(struct evhttp_request *req, void *arg) +{ + struct worker_statistic_info *statistic=(struct worker_statistic_info *)arg; + struct evkeyvalq params; + const char *version, *tablename, *content_range; + int64_t verlong; + char *endptr=NULL; + size_t req_start=0, req_end=0; + + statistic->statistic.field[DRS_FSSTAT_CLIENT_FILE_REQ] += 1; + if(evhttp_parse_query(evhttp_request_get_uri(req), ¶ms)) + { + statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1; + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid"); + return; + } + if(NULL==(version=evhttp_find_header(¶ms, "version")) || NULL==(tablename=evhttp_find_header(¶ms, "tablename"))) + { + statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1; + evhttp_send_error(req, HTTP_BADREQUEST, "Parameters invalid, no version/tablename found"); + return; + } + if(0==(verlong = strtol(version, &endptr, 10)) || *endptr!='\0') + { + statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1; + evhttp_send_error(req, HTTP_BADREQUEST, "Parameter version invalid"); + return; + } + if(NULL!=(content_range = evhttp_find_header(evhttp_request_get_input_headers(req), "Range")) && + sscanf(content_range, "%*[^0-9]%lu-%lu", &req_start, &req_end)<1) + { + statistic->statistic.field[DRS_FSSTAT_CLIENT_INVALID_REQ] += 1; + evhttp_send_error(req, HTTP_BADREQUEST, "Header Range invalid"); + return; + } + + doris_response_file_range(req, tablename, verlong, req_start, req_end, (content_range==NULL)?false:true, statistic); +} + +void doris_http_server_generic_cb(struct evhttp_request *req, void *arg) +{ + evhttp_send_error(req, HTTP_BADREQUEST, "Not Supported."); +} + +void* thread_doris_http_server(void *arg) +{ + struct event_base *worker_evbase; + struct evhttp *worker_http; + struct worker_statistic_info statistic; + struct timeval tv; + + prctl(PR_SET_NAME, "http_server"); + + memset(&statistic, 0, sizeof(struct worker_statistic_info)); + worker_evbase = event_base_new(); + + worker_http = evhttp_new(worker_evbase); + + evhttp_set_cb(worker_http, "/configmeta", doris_http_server_meta_cb, &statistic); + evhttp_set_cb(worker_http, "/configfile", doris_http_server_file_cb, &statistic); + evhttp_set_gencb(worker_http, doris_http_server_generic_cb, &statistic); + evhttp_set_allowed_methods(worker_http, EVHTTP_REQ_GET|EVHTTP_REQ_HEAD); + + if(evhttp_accept_socket(worker_http, g_doris_server_info.listener)) + { + printf("evhttp_accept_socket %d error!\n", g_doris_server_info.listener); + assert(0); return NULL; + } + + evtimer_assign(&statistic.timer_statistic, worker_evbase, doris_worker_statistic_timer_cb, &statistic); + tv.tv_sec = g_doris_server_info.fsstat_period; + tv.tv_usec = 0; + evtimer_add(&statistic.timer_statistic, &tv); + + event_base_dispatch(worker_evbase); + printf("Libevent dispath error, should not run here.\n"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Libevent dispath error, should not run here."); + return NULL; +} + diff --git a/server/doris_server_http.h b/server/doris_server_http.h new file mode 100644 index 0000000..755df29 --- /dev/null +++ b/server/doris_server_http.h @@ -0,0 +1,8 @@ +#ifndef __DORIS_SERVER_HTTP_H__ +#define __DORIS_SERVER_HTTP_H__ + +int doris_create_listen_socket(int bind_port); +void* thread_doris_http_server(void *arg); + +#endif + diff --git a/server/doris_server_main.cpp b/server/doris_server_main.cpp new file mode 100644 index 0000000..b8a1301 --- /dev/null +++ b/server/doris_server_main.cpp @@ -0,0 +1,306 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "doris_client.h" +#include "doris_server_main.h" +#include "doris_server_http.h" + +struct nirvana_global_info g_doris_server_info; +static unsigned long doris_vesion_20210716=20210716L; + +int doris_mkdir_according_path(const char * path) +{ + char buffer[256]; + const char *ps=path, *pc; + + if(*ps == '/') + ps += 1; + + while((pc = strchr(ps, '/')) != NULL) + { + while(*(pc+1) == '/') + pc++; + + memcpy(buffer, path, pc - path); + buffer[pc-path] = '\0'; + + if(access(buffer, F_OK)) + { + if(mkdir(buffer, 0777)) + { + return -1; + } + } + ps = pc + 1; + } + if(access(path, F_OK)) + { + if(mkdir(path, 0777)) + { + return -1; + } + } + return 0; +} + +int32_t doris_read_profile_configs(const char *config_file) +{ + char tmp_buf[4096], tmp_dir[512], tmp_dir2[512]; + + MESA_load_profile_string_def(config_file, "DORIS_SERVER", "run_log_dir", g_doris_server_info.root_log_dir, sizeof(g_doris_server_info.root_log_dir), "./log"); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "run_log_lv", &g_doris_server_info.log_level, 10); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "statistic_log_interval", &g_doris_server_info.statistic_period, 300); + /*runtimelog*/ + snprintf(tmp_dir, 256, "%s/runtime_log", g_doris_server_info.root_log_dir); + if(doris_mkdir_according_path(tmp_dir)) + { + printf("mkdir %s for duran runtimelog failed: %s\n", tmp_dir, strerror(errno)); + return -1; + } + snprintf(tmp_buf, 256, "%s/nirvana_runtime.log", tmp_dir); + g_doris_server_info.log_runtime = MESA_create_runtime_log_handle(tmp_buf, g_doris_server_info.log_level); + if(NULL==g_doris_server_info.log_runtime) + { + printf("MESA_create_runtime_log_handle %s failed: %s\n", tmp_buf, strerror(errno)); + return -1; + } + + /*System*/ + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "worker_thread_num", &g_doris_server_info.iothreads, 1); + if(g_doris_server_info.iothreads > 255) + { + printf("%s: [DORIS_SERVER]worker_thread_num=%d should not be bigger than 255!", config_file, g_doris_server_info.iothreads); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]worker_thread_num=%d should not be bigger than 255!", config_file, g_doris_server_info.iothreads); + assert(0);return -1; + } + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "cache_file_frag_size", &g_doris_server_info.cache_frag_size, 67108864); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "doris_server_role_on", &g_doris_server_info.server_role_sw, 1); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "doris_write_file_on", &g_doris_server_info.write_file_sw, 1); + + if(0>MESA_load_profile_string_nodef(config_file, "DORIS_SERVER", "store_config_path", g_doris_server_info.store_path_root, sizeof(g_doris_server_info.store_path_root))) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]store_config_path not found!", config_file); + assert(0);return -1; + } + snprintf(tmp_dir, 512, "%s/full/index", g_doris_server_info.store_path_root); + snprintf(tmp_dir2,512, "%s/inc/index", g_doris_server_info.store_path_root); + if(doris_mkdir_according_path(tmp_dir) || doris_mkdir_according_path(tmp_dir2)) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "mkdir %s failed: %s\n", tmp_dir, strerror(errno)); + return -1; + } + + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "receive_config_way", &g_doris_server_info.recv_way, RECV_WAY_DRS_CLIENT); + if(g_doris_server_info.recv_way == RECV_WAY_IDX_FILE) + { + if(0>MESA_load_profile_string_nodef(config_file, "DORIS_SERVER", "receive_config_path_full", g_doris_server_info.recv_path_full, sizeof(g_doris_server_info.recv_path_full))) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]receive_config_path not found!", config_file); + assert(0);return -1; + } + if(0>MESA_load_profile_string_nodef(config_file, "DORIS_SERVER", "receive_config_path_inc", g_doris_server_info.recv_path_inc, sizeof(g_doris_server_info.recv_path_inc))) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]receive_config_path not found!", config_file); + assert(0);return -1; + } + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "scan_index_file_interval", &g_doris_server_info.scan_idx_interval, 10); + } + + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "ssl_tcp_connection_on", &g_doris_server_info.ssl_conn_on, 0); + if(g_doris_server_info.ssl_conn_on) + { + MESA_load_profile_string_def(config_file, "DORIS_SERVER", "ssl_trusted_ca_path", g_doris_server_info.ssl_CA_path, 256, "./conf/ssl_CA_path/"); + MESA_load_profile_string_def(config_file, "DORIS_SERVER", "ssl_certificate_file", g_doris_server_info.ssl_cert_file, 256, "./conf/server_cert.pem"); + MESA_load_profile_string_def(config_file, "DORIS_SERVER", "ssl_private_key_file", g_doris_server_info.ssl_key_file, 256, "./conf/server_key.pem"); + MESA_load_profile_string_def(config_file, "DORIS_SERVER", "ssl_private_key_passwd", g_doris_server_info.ssl_key_passwd, 64, "12345678"); + } + + /*FiledStat*/ + MESA_load_profile_string_def(config_file, "DORIS_SERVER", "fsstat_log_appname", g_doris_server_info.fsstat_appname, 16, "DORIS_SERVER_S"); + MESA_load_profile_string_def(config_file, "DORIS_SERVER", "fsstat_log_filepath", g_doris_server_info.fsstat_filepath, 256, "./log/nirvana_server.fs"); + MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "fsstat_log_interval", &g_doris_server_info.fsstat_period, 10); + MESA_load_profile_int_def(config_file, "DORIS_SERVER", "fsstat_log_print_mode", &g_doris_server_info.fsstat_print_mode, 1); + MESA_load_profile_string_def(config_file, "DORIS_SERVER", "fsstat_log_dst_ip", g_doris_server_info.fsstat_dst_ip, 64, "127.0.0.1"); + MESA_load_profile_int_def(config_file, "DORIS_SERVER", "fsstat_log_dst_port", &g_doris_server_info.fsstat_dst_port, 8125); + + //LIBEVENT + MESA_load_profile_int_def(config_file, "DORIS_SERVER", "server_listen_port", &g_doris_server_info.server_port, 9898); + MESA_load_profile_int_def(config_file, "DORIS_SERVER", "manage_listen_port", &g_doris_server_info.manager_port, 2233); + MESA_load_profile_int_def(config_file, "DORIS_SERVER", "socket_recv_bufsize", &g_doris_server_info.sock_recv_bufsize, 524288); + return 0; +} + +static int doris_server_register_field_stat(struct nirvana_global_info *param) +{ + const char *field_names[DRS_FSSTAT_FIELD_NUM]={"RecvFullVer", "RecvIncVer", "RecvErrVer", "FileStarts", "FileComplete", + "ClientInvReq", "ClientMetaReq", "SendResMeta", "SendNoNewMeta", "ClientFileReq", "SendFiles", "SendBytes", "SendFile404"}; + const char *status_names[DRS_FSSTAT_STATUS_NUM]={"MemoryUsed", "CurFullVer", "CurIncVer", "TotalCfgNum"}; + int value; + + param->fsstat_handle = FS_create_handle(); + FS_set_para(param->fsstat_handle, OUTPUT_DEVICE, param->fsstat_filepath, strlen(param->fsstat_filepath)+1); + if(param->fsstat_print_mode == 1) + { + FS_set_para(param->fsstat_handle, PRINT_MODE, ¶m->fsstat_print_mode, sizeof(param->fsstat_print_mode)); + } + else + { + FS_set_para(param->fsstat_handle, PRINT_MODE, ¶m->fsstat_print_mode, sizeof(param->fsstat_print_mode)); + value = 1; + FS_set_para(param->fsstat_handle, FLUSH_BY_DATE, &value, sizeof(value)); + } + value = param->fsstat_period; + FS_set_para(param->fsstat_handle, STAT_CYCLE, &value, sizeof(value)); + value = 0; + FS_set_para(param->fsstat_handle, CREATE_THREAD, &value, sizeof(value)); + FS_set_para(param->fsstat_handle, APP_NAME, param->fsstat_appname, strlen(param->fsstat_appname)+1); + FS_set_para(param->fsstat_handle, STATS_SERVER_IP, param->fsstat_dst_ip, strlen(param->fsstat_dst_ip)+1); + FS_set_para(param->fsstat_handle, STATS_SERVER_PORT, ¶m->fsstat_dst_port, sizeof(param->fsstat_dst_port)); + + for(int i=0; ifsstat_field[i] = FS_register(param->fsstat_handle, FS_STYLE_FIELD, FS_CALC_CURRENT, field_names[i]); + } + for(int i=0; ifsstat_status[i] = FS_register(param->fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, status_names[i]); + } + FS_start(param->fsstat_handle); + return 0; +} + +static void instance_fsstat_output_timer_cb(int fd, short kind, void *userp) +{ + struct timeval tv; + + FS_passive_output(g_doris_server_info.fsstat_handle); + + tv.tv_sec = g_doris_server_info.fsstat_period; + tv.tv_usec = 0; + event_add((struct event*)userp, &tv); +} + +static void manager_statistic_threads_requests_cb(struct evhttp_request *req, void *arg) +{ + evhttp_send_error(req, HTTP_BADREQUEST, "Not Supported."); +} + +static void manager_statistic_status_requests_cb(struct evhttp_request *req, void *arg) +{ + evhttp_send_error(req, HTTP_BADREQUEST, "Not Supported."); +} + +static void manager_statistic_field_requests_cb(struct evhttp_request *req, void *arg) +{ + evhttp_send_error(req, HTTP_BADREQUEST, "Not Supported."); +} + +void manager_generic_requests_cb(struct evhttp_request *req, void *arg) +{ + evhttp_send_error(req, HTTP_BADREQUEST, "Not Supported."); +} + +int main(int argc, char **argv) +{ + struct event_base *manage_evbase; + pthread_t thread_desc; + pthread_attr_t attr; + struct timeval tv; + struct event timer_statistic_fsstat; + struct evhttp *manager_http; + + if(doris_read_profile_configs(NIRVANA_CONFIG_FILE) || doris_server_register_field_stat(&g_doris_server_info)) + { + return -1; + } + g_doris_server_info.cfgver_head = config_version_handle_new(); + pthread_rwlock_init(&g_doris_server_info.rwlock, NULL); + evthread_use_pthreads(); + manage_evbase = event_base_new(); + + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + if(g_doris_server_info.recv_way == RECV_WAY_DRS_CLIENT) + { + if(pthread_create(&thread_desc, &attr, thread_doris_client_recv_cfg, manage_evbase)) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "pthread_create(): %s", strerror(errno)); + assert(0);return -4; + } + } + else + { + if(pthread_create(&thread_desc, &attr, thread_index_file_recv_cfg, NULL)) + { + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "pthread_create(): %s", strerror(errno)); + assert(0);return -4; + } + } + + /*Doris manager server*/ + g_doris_server_info.manager = doris_create_listen_socket(g_doris_server_info.manager_port); + if(g_doris_server_info.manager < 0) + { + return -6; + } + if(NULL == (manager_http = evhttp_new(manage_evbase))) + { + printf("evhttp_new error!\n"); + assert(0); return -2; + } + evhttp_set_cb(manager_http, "/doris/statistic/field", manager_statistic_field_requests_cb, NULL); + evhttp_set_cb(manager_http, "/doris/statistic/status", manager_statistic_status_requests_cb, NULL); + evhttp_set_cb(manager_http, "/doris/statistic/threads", manager_statistic_threads_requests_cb, NULL); + evhttp_set_gencb(manager_http, manager_generic_requests_cb, NULL); + g_doris_server_info.monitor = MESA_Monitor_instance_evhttp_new(manager_http, doris_vesion_20210716); + g_doris_server_info.mm_latest_ver = MESA_Monitor_register(g_doris_server_info.monitor, "latest_cfg_version", MONITOR_METRICS_GAUGE, "Latest doris config version."); + g_doris_server_info.mm_total_cfgnum = MESA_Monitor_register(g_doris_server_info.monitor, "total_config_num", MONITOR_METRICS_GAUGE, "Total config num from latest full version till now."); + + if(evhttp_accept_socket(manager_http, g_doris_server_info.manager)) + { + printf("evhttp_accept_socket %d error!\n", g_doris_server_info.manager); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "evhttp_accept_socket %d error!\n", g_doris_server_info.manager); + assert(0); return -7; + } + + /*Doris http server*/ + if(g_doris_server_info.server_role_sw) + { + g_doris_server_info.listener = doris_create_listen_socket(g_doris_server_info.server_port); + if(g_doris_server_info.listener < 0) + { + return -5; + } + for(u_int32_t i=0; i +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "MESA_Monitor.h" + +#include "doris_server_receive.h" + +#ifndef __FILENAME__ +#define __FILENAME__ __FILE__ +#endif +#define MESA_RUNTIME_LOGV3(handle, lv, fmt, args...) \ + MESA_handle_runtime_log((handle), (lv), "DorisServer", "%s:%d, " fmt, __FILENAME__, __LINE__, ##args) + +#define NIRVANA_CONFIG_FILE "./conf/doris_main.conf" + +#define RECV_WAY_DRS_CLIENT 1 +#define RECV_WAY_IDX_FILE 2 + +struct nirvana_global_info +{ + u_int32_t iothreads; + int32_t server_port; + int32_t manager_port; + int32_t sock_recv_bufsize; + u_int32_t ssl_conn_on; + u_int32_t recv_way; + char recv_path_full[256]; + char recv_path_inc[256]; + char store_path_root[256]; + char store_path_inc[256]; + u_int32_t scan_idx_interval; + u_int32_t cache_frag_size; + u_int32_t server_role_sw; + u_int32_t write_file_sw; + + char ssl_CA_path[256]; + char ssl_cert_file[256]; + char ssl_key_file[256]; + char ssl_key_passwd[64]; + pthread_mutex_t *lock_cs; + SSL_CTX *ssl_instance; + + struct version_list_handle *cfgver_head; + evutil_socket_t listener; + evutil_socket_t manager; + pthread_rwlock_t rwlock; + + struct MESA_MonitorHandler *monitor; + int32_t mm_latest_ver; + int32_t mm_total_cfgnum; + + /*logs*/ + u_int32_t log_level; + u_int32_t statistic_period; + char root_log_dir[256]; + void *log_runtime; + + screen_stat_handle_t fsstat_handle; + char fsstat_dst_ip[64]; + char fsstat_appname[16]; + char fsstat_filepath[256]; + u_int32_t fsstat_period; + int32_t fsstat_print_mode; + int32_t fsstat_dst_port; + int32_t fsstat_field[DRS_FSSTAT_FIELD_NUM]; + int32_t fsstat_status[DRS_FSSTAT_STATUS_NUM]; +}; + +int doris_mkdir_according_path(const char * path); + +#endif + diff --git a/server/doris_server_receive.cpp b/server/doris_server_receive.cpp new file mode 100644 index 0000000..113c0e0 --- /dev/null +++ b/server/doris_server_receive.cpp @@ -0,0 +1,739 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "doris_server_main.h" +#include "doris_server_scandir.h" +#include "doris_server_receive.h" + +struct scanner_timer_priv +{ + struct doris_callbacks doris_cbs; + struct doris_arguments doris_args; + struct doris_idxfile_scanner *scanner; + struct event timer_scanner; +}; + +extern struct nirvana_global_info g_doris_server_info; + + +void doris_worker_statistic_timer_cb(int fd, short kind, void *userp) +{ + struct worker_statistic_info *statistic = (struct worker_statistic_info *)userp; + struct timeval tv; + struct doris_srv_statistics incr_statistic; + long long *plast_statistic = (long long*)&statistic->statistic_last; + long long *pnow_statistic = (long long*)&statistic->statistic; + long long *pinc_statistic = (long long*)&incr_statistic; + + for(u_int32_t i=0; istatistic_last = statistic->statistic; + + for(u_int32_t i=0; itimer_statistic, &tv); +} + +void config_frag_node_cleanup(struct confile_save *save, struct cont_frag_node *fragnode) +{ + if(fragnode == NULL) return; + + save->statistic.statistic.status[DRS_FSSTAT_MEMORY_USED] -= fragnode->totalsize; + free(fragnode->content); + free(fragnode); +} + +void config_table_node_cleanup(struct confile_save *save, struct table_list_node *table_node) +{ + struct cont_frag_node *fragnode; + + if(table_node == NULL) return; + + while(NULL != (fragnode = TAILQ_FIRST(&table_node->frag_head))) + { + TAILQ_REMOVE(&table_node->frag_head, fragnode, frag_node); + config_frag_node_cleanup(save, fragnode); + } + free(table_node); +} + +void config_version_node_cleanup(struct confile_save *save, struct version_list_node *vernode) +{ + struct table_list_node *tablenode; + + if(vernode == NULL) return; + + while(NULL != (tablenode = TAILQ_FIRST(&vernode->table_head))) + { + TAILQ_REMOVE(&vernode->table_head, tablenode, table_node); + config_table_node_cleanup(save, tablenode); + } + free(vernode->metacont); + cJSON_Delete(vernode->metajson); + cJSON_Delete(vernode->arrayjson); + free(vernode); +} + +void config_version_handle_cleanup(struct confile_save *save, struct version_list_handle *version) +{ + struct version_list_node *vernode; + + while(NULL != (vernode = TAILQ_FIRST(&version->version_head))) + { + TAILQ_REMOVE(&version->version_head, vernode, version_node); + config_version_node_cleanup(save, vernode); + } + free(version); +} + +struct version_list_handle *config_version_handle_new(void) +{ + struct version_list_handle *handle; + + handle = (struct version_list_handle *)calloc(1, sizeof(struct version_list_handle)); + TAILQ_INIT(&handle->version_head); + return handle; +} + +static void doris_common_timer_start(struct event *time_event) +{ + struct timeval tv; + + tv.tv_sec = 2; + tv.tv_usec = 0; + evtimer_add(time_event, &tv); +} + +static void cfgver_delay_destroy_timer_cb(int fd, short kind, void *userp) +{ + struct common_timer_event *delay_event=(struct common_timer_event *)userp; + struct version_list_handle *handle = (struct version_list_handle *)delay_event->data; + + if(handle->references != 0) + { + doris_common_timer_start(&delay_event->timer_event); + return; + } + config_version_handle_cleanup(delay_event->save, handle); + free(delay_event); +} + +static void cfgver_handle_delay_destroy(struct confile_save *save, struct event_base *evbase, struct version_list_handle *version) +{ + struct common_timer_event *delay_event; + + delay_event = (struct common_timer_event *)malloc(sizeof(struct common_timer_event)); + delay_event->data = version; + delay_event->save = save; + evtimer_assign(&delay_event->timer_event, evbase, cfgver_delay_destroy_timer_cb, delay_event); + doris_common_timer_start(&delay_event->timer_event); +} + +/*file系列函数,写本地文件*/ +void doris_config_file_version_start(struct doris_instance *instance, cJSON *meta, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + + if(save->type == CFG_UPDATE_TYPE_FULL) + { + snprintf(save->inc_index_path, 512, "%s/inc/index/full_config_index.%010lu", g_doris_server_info.store_path_root, save->version); + snprintf(save->tmp_index_path, 512, "%s/inc/full_config_index.%010lu.ing", g_doris_server_info.store_path_root, save->version); + snprintf(save->full_index_path, 512, "%s/full/index/full_config_index.%010lu", g_doris_server_info.store_path_root, save->version); + } + else + { + snprintf(save->inc_index_path, 512, "%s/inc/index/inc_config_index.%010lu", g_doris_server_info.store_path_root, save->version); + snprintf(save->tmp_index_path, 512, "%s/inc/full_config_index.%010lu.ing", g_doris_server_info.store_path_root, save->version); + } + save->fp_idx_file = fopen(save->tmp_index_path, "w+"); +} + +void doris_config_file_version_finish(struct doris_instance *instance, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + + fclose(save->fp_idx_file); + if(rename(save->tmp_index_path, save->inc_index_path)) + { + assert(0); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "rename %s to %s failed: %s", save->tmp_index_path, save->inc_index_path, strerror(errno)); + } + if(save->type == CFG_UPDATE_TYPE_FULL) + { + if(link(save->inc_index_path, save->full_index_path) && errno!=EEXIST) //创建硬链接 + { + assert(0); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "rename %s to %s failed: %s", save->tmp_index_path, save->inc_index_path, strerror(errno)); + } + } + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "Version %lu write finished, index file: %s", save->version, save->inc_index_path); +} + +void doris_config_file_version_error(struct doris_instance *instance, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + + if(save->fp_idx_file != NULL) + { + fclose(save->fp_idx_file); + remove(save->tmp_index_path); + } + if(save->fp_cfg_file != NULL) + { + fclose(save->fp_cfg_file); + remove(save->cfg_file_path); + } + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Version %llu error, rolling back...", save->version); +} + +void doris_config_file_cfgfile_start(struct doris_instance *instance, const char *tablename, size_t size, u_int32_t cfgnum, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + struct tm *localtm, savetime; + time_t now; + const char *type; + char dir[256]; + + type = (save->type == CFG_UPDATE_TYPE_FULL)?"full":"inc"; + now = time(NULL); + localtm = localtime_r(&now, &savetime); + snprintf(dir, 256, "%s/%s/%04d-%02d-%02d", g_doris_server_info.store_path_root, type, localtm->tm_year+1900, localtm->tm_mon+1, localtm->tm_mday); + if(access(dir, F_OK)) + { + doris_mkdir_according_path(dir); + } + snprintf(save->cfg_file_path, 256, "%s/%s.%010lu", dir, tablename, save->version); + fprintf(save->fp_idx_file, "%s\t%u\t%s\n", tablename, cfgnum, save->cfg_file_path); + + save->fp_cfg_file = fopen(save->cfg_file_path, "w+"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "File %s start writing...", save->cfg_file_path); +} + + +void doris_config_file_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + size_t writen_len; + + writen_len = fwrite(data, 1, len, save->fp_cfg_file); + assert(writen_len==len); +} + +void doris_config_file_cfgfile_finish(struct doris_instance *instance, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + fclose(save->fp_cfg_file); + + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "File %s write finished", save->cfg_file_path); +} + +/*mem系列函数,加载入内存*/ +void doris_config_mem_version_start(struct doris_instance *instance, cJSON *meta, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + + save->cur_vernode = (struct version_list_node *)calloc(1, sizeof(struct version_list_node)); + TAILQ_INIT(&save->cur_vernode->table_head); + save->cur_vernode->metajson = cJSON_CreateObject(); + save->cur_vernode->arrayjson= cJSON_CreateArray(); + + save->cur_vernode->version = save->version; + cJSON_AddNumberToObject(save->cur_vernode->metajson, "version", save->cur_vernode->version); + + save->cur_vernode->cfg_type = save->type; + cJSON_AddNumberToObject(save->cur_vernode->metajson, "type", save->cur_vernode->cfg_type); +} + +void doris_config_mem_version_finish(struct doris_instance *instance, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + struct version_list_handle *cur_version; + struct version_list_handle *tmplist; + + cJSON_AddItemToObject(save->cur_vernode->metajson, "configs", save->cur_vernode->arrayjson); + save->cur_vernode->arrayjson = NULL; + save->cur_vernode->metacont = cJSON_PrintUnformatted(save->cur_vernode->metajson); + save->cur_vernode->metalen = strlen(save->cur_vernode->metacont); + cJSON_Delete(save->cur_vernode->metajson); + save->cur_vernode->metajson = NULL; + + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Version %lu mem finished, info: %s", save->version, save->cur_vernode->metacont); + + if(save->cur_vernode->cfg_type==CFG_UPDATE_TYPE_FULL && g_doris_server_info.cfgver_head->latest_version!=0) + { + cur_version = config_version_handle_new(); + cur_version->latest_version = save->cur_vernode->version; + TAILQ_INSERT_TAIL(&cur_version->version_head, save->cur_vernode, version_node); + + pthread_rwlock_wrlock(&g_doris_server_info.rwlock); + tmplist = g_doris_server_info.cfgver_head; + g_doris_server_info.cfgver_head = cur_version; + pthread_rwlock_unlock(&g_doris_server_info.rwlock); + cfgver_handle_delay_destroy(save, save->evbase, tmplist); + } + else + { + TAILQ_INSERT_TAIL(&g_doris_server_info.cfgver_head->version_head, save->cur_vernode, version_node); + g_doris_server_info.cfgver_head->latest_version = save->cur_vernode->version; + } + save->cur_vernode = NULL; +} + +void doris_config_mem_version_error(struct doris_instance *instance, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + + config_frag_node_cleanup(save, save->cur_frag); + config_table_node_cleanup(save, save->cur_table); + config_version_node_cleanup(save, save->cur_vernode); + save->cur_frag = NULL; + save->cur_table = NULL; + save->cur_vernode = NULL; +} + +void doris_config_mem_cfgfile_start(struct doris_instance *instance, const char *tablename, size_t size, u_int32_t cfgnum, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + cJSON *table_meta; + + table_meta = cJSON_CreateObject(); + cJSON_AddStringToObject(table_meta, "tablename", tablename); + cJSON_AddNumberToObject(table_meta, "cfg_num", cfgnum); + cJSON_AddNumberToObject(table_meta, "size", size); + cJSON_AddItemToArray(save->cur_vernode->arrayjson, table_meta); + + save->cur_table = (struct table_list_node *)calloc(1, sizeof(struct table_list_node)); + snprintf(save->cur_table->tablename, 64, "%s", tablename); + save->cur_table->filesize = size; + TAILQ_INIT(&save->cur_table->frag_head); + + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "table %s.%010llu start loading to memory...", tablename, save->version); +} + +void doris_config_mem_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + size_t cache_len, offset=0; + + save->statistic.statistic.status[DRS_FSSTAT_MEMORY_USED] += len; + while(len > 0) + { + if(save->cur_frag == NULL) + { + save->cur_frag = (struct cont_frag_node *)calloc(1, sizeof(struct cont_frag_node)); + save->cur_frag->start = save->cur_table->cur_totallen; + save->cur_frag->totalsize = save->cur_table->filesize - save->cur_table->cur_totallen; + if(save->cur_frag->totalsize > g_doris_server_info.cache_frag_size) + { + save->cur_frag->totalsize = g_doris_server_info.cache_frag_size; + } + save->cur_frag->end = save->cur_frag->start + save->cur_frag->totalsize - 1; + save->cur_frag->content = (char *)malloc(save->cur_frag->totalsize); + } + + if(save->cur_frag->totalsize > save->cur_frag->cur_fraglen + len) + { + memcpy(save->cur_frag->content+save->cur_frag->cur_fraglen, data+offset, len); + save->cur_frag->cur_fraglen += len; + save->cur_table->cur_totallen += len; + offset += len; + len = 0; + } + else + { + cache_len = save->cur_frag->totalsize - save->cur_frag->cur_fraglen; + memcpy(save->cur_frag->content+save->cur_frag->cur_fraglen, data+offset, cache_len); + save->cur_frag->cur_fraglen += cache_len; + save->cur_table->cur_totallen += cache_len; + offset += cache_len; + len -= cache_len; + TAILQ_INSERT_TAIL(&save->cur_table->frag_head, save->cur_frag, frag_node); + assert(save->cur_frag->cur_fraglen == save->cur_frag->end - save->cur_frag->start + 1); + save->cur_frag = NULL; + } + } + assert(save->cur_table->cur_totallen <= save->cur_table->filesize); +} + +void doris_config_mem_cfgfile_finish(struct doris_instance *instance, void *userdata) +{ + struct confile_save *save=(struct confile_save *)userdata; + + if(save->cur_frag != NULL) + { + TAILQ_INSERT_TAIL(&save->cur_table->frag_head, save->cur_frag, frag_node); + assert(save->cur_frag->cur_fraglen == save->cur_frag->end - save->cur_frag->start + 1); + save->cur_frag = NULL; + } + assert(save->cur_table->cur_totallen == save->cur_table->filesize); + TAILQ_INSERT_TAIL(&save->cur_vernode->table_head, save->cur_table, table_node); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "table %s.%010llu load to memory finished", save->cur_table->tablename, save->version); + save->cur_table = NULL; +} + +/*common系列函数,载入配置时公共操作*/ +void doris_config_common_version_start(struct confile_save *save, cJSON *meta) +{ + cJSON *sub; + + sub = cJSON_GetObjectItem(meta, "version"); + save->version = sub->valuedouble; + sub = cJSON_GetObjectItem(meta, "type"); + save->type = sub->valueint; + assert(save->type==CFG_UPDATE_TYPE_FULL || save->type==CFG_UPDATE_TYPE_INC); + save->version_cfgnum = 0; + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "Version %lu start updating...", save->version); +} + +void doris_config_common_version_finish(struct confile_save *save) +{ + if(save->type == CFG_UPDATE_TYPE_FULL) + { + save->statistic.statistic.status[DRS_FSSTAT_CUR_FULL_VERSION] = save->version; + save->statistic.statistic.status[DRS_FSSTAT_CONFIG_TOTAL_NUM] = save->version_cfgnum; + save->statistic.statistic.field[DRS_FSSTAT_RECV_FULL_VER] += 1; + } + else + { + save->statistic.statistic.status[DRS_FSSTAT_CUR_INC_VERSION] = save->version; + save->statistic.statistic.status[DRS_FSSTAT_CONFIG_TOTAL_NUM] += save->version_cfgnum; + save->statistic.statistic.field[DRS_FSSTAT_RECV_INC_VER] += 1; + } + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mm_latest_ver, MONITOR_VALUE_SET, save->version); + MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mm_total_cfgnum, MONITOR_VALUE_SET, + save->statistic.statistic.status[DRS_FSSTAT_CONFIG_TOTAL_NUM]); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_INFO, "Version %lu update finished", save->version); + } + +void doris_config_common_version_error(struct confile_save *save) +{ + save->statistic.statistic.field[DRS_FSSTAT_RECV_ERR_VER] += 1; +} + +void doris_config_common_cfgfile_start(struct confile_save *save, u_int32_t cfgnum) +{ + save->version_cfgnum += cfgnum; + save->statistic.statistic.field[DRS_FSSTAT_RECV_START_FILES] += 1; +} + +void doris_config_common_cfgfile_finish(struct confile_save *save) +{ + save->statistic.statistic.field[DRS_FSSTAT_RECV_CMPLT_FILES] += 1; +} + +/*localmem系列函数,启动时从本地缓存读的回调*/ +void doris_config_localmem_version_start(struct doris_instance *instance, cJSON *meta, void *userdata) +{ + doris_config_common_version_start((struct confile_save *)userdata, meta); + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_version_start(instance, meta, userdata); + } +} + +void doris_config_localmem_version_finish(struct doris_instance *instance, void *userdata) +{ + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_version_finish(instance, userdata); + } + doris_config_common_version_finish((struct confile_save *)userdata); +} + +void doris_config_localmem_version_error(struct doris_instance *instance, void *userdata) +{ + doris_config_common_version_error((struct confile_save *)userdata); + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_version_error(instance, userdata); + } +} + +void doris_config_localmem_cfgfile_start(struct doris_instance *instance, const char *tablename, size_t size, u_int32_t cfgnum, void *userdata) +{ + doris_config_common_cfgfile_start((struct confile_save *)userdata, cfgnum); + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_cfgfile_start(instance, tablename, size, cfgnum, userdata); + } +} + +void doris_config_localmem_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata) +{ + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_cfgfile_update(instance, data, len, userdata); + } +} + +void doris_config_localmem_cfgfile_finish(struct doris_instance *instance, void *userdata) +{ + doris_config_common_cfgfile_finish((struct confile_save *)userdata); + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_cfgfile_finish(instance, userdata); + } +} + +/*无标记系列函数,新来配置时回调*/ +void doris_config_version_start(struct doris_instance *instance, cJSON *meta, void *userdata) +{ + doris_config_common_version_start((struct confile_save *)userdata, meta); + if(g_doris_server_info.write_file_sw) + { + doris_config_file_version_start(instance, meta, userdata); + } + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_version_start(instance, meta, userdata); + } +} + +void doris_config_version_finish(struct doris_instance *instance, void *userdata) +{ + if(g_doris_server_info.write_file_sw) + { + doris_config_file_version_finish(instance, userdata); + } + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_version_finish(instance, userdata); + } + doris_config_common_version_finish((struct confile_save *)userdata); +} + +void doris_config_version_error(struct doris_instance *instance, void *userdata) +{ + doris_config_common_version_error((struct confile_save *)userdata); + if(g_doris_server_info.write_file_sw) + { + doris_config_file_version_error(instance, userdata); + } + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_version_error(instance, userdata); + } +} + +void doris_config_cfgfile_start(struct doris_instance *instance, const char *tablename, size_t size, u_int32_t cfgnum, void *userdata) +{ + doris_config_common_cfgfile_start((struct confile_save *)userdata, cfgnum); + if(g_doris_server_info.write_file_sw) + { + doris_config_file_cfgfile_start(instance, tablename, size, cfgnum, userdata); + } + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_cfgfile_start(instance, tablename, size, cfgnum, userdata); + } +} + +void doris_config_cfgfile_update(struct doris_instance *instance, const char *data, size_t len, void *userdata) +{ + if(g_doris_server_info.write_file_sw) + { + doris_config_file_cfgfile_update(instance, data, len, userdata); + } + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_cfgfile_update(instance, data, len, userdata); + } +} + +void doris_config_cfgfile_finish(struct doris_instance *instance, void *userdata) +{ + doris_config_common_cfgfile_finish((struct confile_save *)userdata); + if(g_doris_server_info.write_file_sw) + { + doris_config_file_cfgfile_finish(instance, userdata); + } + if(g_doris_server_info.server_role_sw) + { + doris_config_mem_cfgfile_finish(instance, userdata); + } +} + +void* thread_doris_client_recv_cfg(void *arg) +{ + struct event_base *manage_evbase=(struct event_base *)arg, *client_evbase; + struct doris_parameter *param; + struct doris_instance *instance; + struct doris_callbacks doris_cbs; + struct doris_arguments doris_args={0, 0, 0}; + struct doris_idxfile_scanner *scanner; + enum DORIS_UPDATE_TYPE update_type; + struct confile_save save; + char stored_path[512]; + struct timeval tv; + + prctl(PR_SET_NAME, "client_recv"); + + client_evbase = event_base_new(); + + memset(&save, 0, sizeof(struct confile_save)); + save.source_from = RECV_WAY_IDX_FILE; + save.evbase = client_evbase; + + scanner = doris_index_file_scanner(0); + + /*Retaive latest config to memory from Stored configs*/ + doris_cbs.version_start = doris_config_localmem_version_start; + doris_cbs.version_finish = doris_config_localmem_version_finish; + doris_cbs.version_error = doris_config_localmem_version_error; + doris_cbs.cfgfile_start = doris_config_localmem_cfgfile_start; + doris_cbs.cfgfile_update = doris_config_localmem_cfgfile_update; + doris_cbs.cfgfile_finish = doris_config_localmem_cfgfile_finish; + doris_cbs.userdata = &save; + + snprintf(stored_path, 512, "%s/full/index", g_doris_server_info.store_path_root); + update_type = doris_index_file_traverse(scanner, stored_path, &doris_cbs, NULL, g_doris_server_info.log_runtime); + assert(update_type!=CFG_UPDATE_TYPE_ERR); + snprintf(stored_path, 512, "%s/inc/index", g_doris_server_info.store_path_root); + do { + update_type = doris_index_file_traverse(scanner, stored_path, &doris_cbs, NULL, g_doris_server_info.log_runtime); + assert(update_type!=CFG_UPDATE_TYPE_ERR); + }while(update_type != CFG_UPDATE_TYPE_NONE); + + + /*Check new configs*/ + doris_cbs.version_start = doris_config_version_start; + doris_cbs.version_finish = doris_config_version_finish; + doris_cbs.version_error = doris_config_version_error; + doris_cbs.cfgfile_start = doris_config_cfgfile_start; + doris_cbs.cfgfile_update = doris_config_cfgfile_update; + doris_cbs.cfgfile_finish = doris_config_cfgfile_finish; + + save.source_from = RECV_WAY_DRS_CLIENT; + doris_args.current_version = scanner->cur_version; + param = doris_parameter_new(NIRVANA_CONFIG_FILE, manage_evbase, &doris_cbs, &doris_args, g_doris_server_info.log_runtime); + if(param == NULL) + { + assert(0);return NULL; + } + + instance = doris_instance_new(param, client_evbase, g_doris_server_info.log_runtime); + if(instance == NULL) + { + assert(0);return NULL; + } + + evtimer_assign(&save.statistic.timer_statistic, client_evbase, doris_worker_statistic_timer_cb, &save.statistic); + tv.tv_sec = g_doris_server_info.fsstat_period; + tv.tv_usec = 0; + evtimer_add(&save.statistic.timer_statistic, &tv); + + event_base_dispatch(client_evbase); + printf("Libevent dispath error, should not run here.\n"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Libevent dispath error, should not run here."); + assert(0);return NULL; +} + +static void doris_scanner_timer_cb(int fd, short kind, void *userp) +{ + struct scanner_timer_priv *timer_priv=(struct scanner_timer_priv *)userp; + enum DORIS_UPDATE_TYPE update_type; + struct timeval tv; + + do { + update_type = doris_index_file_traverse(timer_priv->scanner, g_doris_server_info.recv_path_inc, + &timer_priv->doris_cbs, NULL, g_doris_server_info.log_runtime); + }while(update_type != CFG_UPDATE_TYPE_NONE); + + tv.tv_sec = g_doris_server_info.scan_idx_interval; + tv.tv_usec = 0; + evtimer_add(&timer_priv->timer_scanner, &tv); +} + +void* thread_index_file_recv_cfg(void *arg) +{ + struct event_base *client_evbase; + struct confile_save save; + struct timeval tv; + struct scanner_timer_priv timer_priv; + enum DORIS_UPDATE_TYPE update_type; + char stored_path[256]; + + prctl(PR_SET_NAME, "index_file"); + + memset(&save, 0, sizeof(struct confile_save)); + memset(&timer_priv, 0, sizeof(struct scanner_timer_priv)); + + client_evbase = event_base_new(); + + save.source_from = RECV_WAY_IDX_FILE; + save.evbase = client_evbase; + + timer_priv.scanner = doris_index_file_scanner(0); + + /*Retaive latest config to memory from Stored configs*/ + timer_priv.doris_cbs.version_start = doris_config_localmem_version_start; + timer_priv.doris_cbs.version_finish = doris_config_localmem_version_finish; + timer_priv.doris_cbs.version_error = doris_config_localmem_version_error; + timer_priv.doris_cbs.cfgfile_start = doris_config_localmem_cfgfile_start; + timer_priv.doris_cbs.cfgfile_update = doris_config_localmem_cfgfile_update; + timer_priv.doris_cbs.cfgfile_finish = doris_config_localmem_cfgfile_finish; + timer_priv.doris_cbs.userdata = &save; + + snprintf(stored_path, 512, "%s/full/index", g_doris_server_info.store_path_root); + update_type = doris_index_file_traverse(timer_priv.scanner, stored_path, &timer_priv.doris_cbs, NULL, g_doris_server_info.log_runtime); + assert(update_type!=CFG_UPDATE_TYPE_ERR); + snprintf(stored_path, 512, "%s/inc/index", g_doris_server_info.store_path_root); + do{ + update_type = doris_index_file_traverse(timer_priv.scanner, stored_path, &timer_priv.doris_cbs, NULL, g_doris_server_info.log_runtime); + assert(update_type!=CFG_UPDATE_TYPE_ERR); + }while(update_type!=CFG_UPDATE_TYPE_NONE && update_type!=CFG_UPDATE_TYPE_ERR); + + + /*Check new configs*/ + timer_priv.doris_cbs.version_start = doris_config_version_start; + timer_priv.doris_cbs.version_finish = doris_config_version_finish; + timer_priv.doris_cbs.version_error = doris_config_version_error; + timer_priv.doris_cbs.cfgfile_start = doris_config_cfgfile_start; + timer_priv.doris_cbs.cfgfile_update = doris_config_cfgfile_update; + timer_priv.doris_cbs.cfgfile_finish = doris_config_cfgfile_finish; + + update_type = doris_index_file_traverse(timer_priv.scanner, g_doris_server_info.recv_path_full, + &timer_priv.doris_cbs, NULL, g_doris_server_info.log_runtime); + assert(update_type!=CFG_UPDATE_TYPE_ERR); + if(update_type!=CFG_UPDATE_TYPE_NONE && update_type!=CFG_UPDATE_TYPE_ERR) + { + tv.tv_sec = 0; + } + else + { + tv.tv_sec = g_doris_server_info.scan_idx_interval; + } + tv.tv_usec = 0; + evtimer_assign(&timer_priv.timer_scanner, client_evbase, doris_scanner_timer_cb, &timer_priv); + evtimer_add(&timer_priv.timer_scanner, &tv); + + evtimer_assign(&save.statistic.timer_statistic, client_evbase, doris_worker_statistic_timer_cb, &save.statistic); + tv.tv_sec = g_doris_server_info.fsstat_period; + tv.tv_usec = 0; + evtimer_add(&save.statistic.timer_statistic, &tv); + + event_base_dispatch(client_evbase); + printf("Libevent dispath error, should not run here.\n"); + MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "Libevent dispath error, should not run here."); + return NULL; +} + diff --git a/server/doris_server_receive.h b/server/doris_server_receive.h new file mode 100644 index 0000000..4de505b --- /dev/null +++ b/server/doris_server_receive.h @@ -0,0 +1,129 @@ +#ifndef __DORIS_SERVER_RECEIVE_H__ +#define __DORIS_SERVER_RECEIVE_H__ + +#include +#include +#include + +#include + + +enum DORIS_SERVER_FS_FILED +{ + DRS_FSSTAT_RECV_FULL_VER=0, + DRS_FSSTAT_RECV_INC_VER, + DRS_FSSTAT_RECV_ERR_VER, + DRS_FSSTAT_RECV_START_FILES, + DRS_FSSTAT_RECV_CMPLT_FILES, + + DRS_FSSTAT_CLIENT_INVALID_REQ, + DRS_FSSTAT_CLIENT_META_REQ, + DRS_FSSTAT_SEND_META_RES, + DRS_FSSTAT_SEND_META_NONEW, + DRS_FSSTAT_CLIENT_FILE_REQ, + DRS_FSSTAT_SEND_FILE_RES, + DRS_FSSTAT_SEND_FILE_BYTES, + DRS_FSSTAT_SEND_FILE_RES_404, + + DRS_FSSTAT_FIELD_NUM, +}; + +enum DORIS_SERVER_FS_STATUS +{ + DRS_FSSTAT_MEMORY_USED=0, + DRS_FSSTAT_CUR_FULL_VERSION, + DRS_FSSTAT_CUR_INC_VERSION, + DRS_FSSTAT_CONFIG_TOTAL_NUM, + + DRS_FSSTAT_STATUS_NUM, +}; + +struct cont_frag_node +{ + size_t start; + size_t end; + size_t totalsize; + size_t cur_fraglen; + char *content; + + TAILQ_ENTRY(cont_frag_node) frag_node; +}; + +struct table_list_node +{ + char tablename[64]; + size_t filesize; + size_t cur_totallen; + + TAILQ_HEAD(__table_cont_node, cont_frag_node) frag_head; + TAILQ_ENTRY(table_list_node) table_node; +}; + +struct version_list_node +{ + int64_t version; + char *metacont; + int32_t metalen; + int32_t cfg_type; //1-full, 2-inc + cJSON *metajson, *arrayjson; + + TAILQ_HEAD(__table_list_node, table_list_node) table_head; + TAILQ_ENTRY(version_list_node) version_node; +}; + +struct version_list_handle +{ + TAILQ_HEAD(__version_list_node, version_list_node) version_head; + int64_t latest_version; + int32_t references; +}; + +struct version_list_handle *config_version_handle_new(void); + +struct doris_srv_statistics +{ + long long field[DRS_FSSTAT_FIELD_NUM]; + long long status[DRS_FSSTAT_STATUS_NUM]; +}; + +struct worker_statistic_info +{ + struct event timer_statistic; + struct doris_srv_statistics statistic, statistic_last; +}; + +struct confile_save +{ + struct event_base *evbase; + int64_t version; + int32_t source_from; + int32_t type; + int64_t version_cfgnum; + char inc_index_path[256]; //inc目录下的增量全量 + char tmp_index_path[256]; //inc目录下的增量全量 + char full_index_path[256]; //full目录下的全量 + char cfg_file_path[256]; + FILE *fp_cfg_file; + FILE *fp_idx_file; + + struct worker_statistic_info statistic; + + struct version_list_node *cur_vernode; + struct table_list_node *cur_table; + struct cont_frag_node *cur_frag; +}; + +struct common_timer_event +{ + struct event timer_event; + struct confile_save *save; + void *data; +}; + +void doris_worker_statistic_timer_cb(int fd, short kind, void *userp); + +void* thread_doris_client_recv_cfg(void *arg); +void* thread_index_file_recv_cfg(void *arg); + +#endif + diff --git a/server/doris_server_scandir.cpp b/server/doris_server_scandir.cpp new file mode 100644 index 0000000..de55e4a --- /dev/null +++ b/server/doris_server_scandir.cpp @@ -0,0 +1,318 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "doris_server_scandir.h" + +#ifndef __FILENAME__ +#define __FILENAME__ __FILE__ +#endif +#define MESA_RUNTIME_LOGV4(handle, lv, fmt, args...) \ + MESA_handle_runtime_log((handle), (lv), "DorisServer", "%s:%d, " fmt, __FILENAME__, __LINE__, ##args) + +//replacement of glibc scandir, to adapt dictator malloc wrap +#define ENLARGE_STEP 1024 +int my_scandir(const char *dir, struct dirent ***namelist, + int(*filter)(const struct dirent *), + int(*compar)(const void *, const void *)) +{ + DIR * od; + int n = 0; + int DIR_ENT_SIZE=ENLARGE_STEP; + struct dirent ** list = NULL; + struct dirent * p; + struct dirent entry,*result; + + if((dir == NULL) || (namelist == NULL) || (od = opendir(dir))==NULL) + return -1; + + list = (struct dirent **)malloc(DIR_ENT_SIZE*sizeof(struct dirent *)); + while(0==readdir_r(od,&entry,&result)) + { + if(result==NULL) + { + break; + } + if( filter && !filter(&entry)) + continue; + + p = (struct dirent *)malloc(sizeof(struct dirent)); + memcpy((void *)p,(void *)(&entry),sizeof(struct dirent)); + list[n] = p; + + n++; + if(n >= DIR_ENT_SIZE) + { + DIR_ENT_SIZE+=ENLARGE_STEP; + list=(struct dirent **)realloc((void*)list,DIR_ENT_SIZE*sizeof(struct dirent *)); + } + } + closedir(od); + *namelist = list; + + if(compar) + { + qsort((void *)*namelist,n,sizeof(struct dirent *),compar); + } + return n; + +} + +int filter_fn(const struct dirent * ent) +{ + // files in xfs are not DT_REG. + // if(ent->d_type != DT_REG) + // return 0; + + return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0|| + strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0); +} + +enum DORIS_UPDATE_TYPE get_new_idx_path(long long current_version, const char *file_dir, void *logger, struct index_path_array **idx_path, int *idx_num) +{ + struct dirent **namelist; + char update_str[32]={0}; + long long latest_ful_version=0,latest_inc_version=0,config_seq=0; + int n=0,i=0, full_file_idx=0,inc_idx_num=0; + enum DORIS_UPDATE_TYPE update_type=CFG_UPDATE_TYPE_NONE; + struct index_path_array *tmpidx_path_array; + struct index_version_array *tmpidx_ver_array; + + n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort); + if(n < 0) + { + MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "scan dir error"); + return update_type; + } + + tmpidx_ver_array = (struct index_version_array*)calloc(sizeof(struct index_version_array), n); + inc_idx_num=0; + for(i=0;id_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) + { + continue; + } + if(strlen(namelist[i]->d_name) > 42) + { + MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "config file %s filename too long,should like full_config_index.00000000000000000001", namelist[i]->d_name); + continue; + } + if(sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld",update_str,&config_seq) != 2) + { + MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "config file %s filename error,should like full_config_index.00000000000000000001", namelist[i]->d_name); + continue; + } + if(strncasecmp(update_str,"full",strlen(update_str))==0) + { + if(config_seq>latest_ful_version) + { + latest_ful_version=config_seq; + full_file_idx=i; + } + } + else if(strncasecmp(update_str,"inc",strlen(update_str))==0) + { + if(config_seq > current_version) + { + tmpidx_ver_array[inc_idx_num].file_inx = i; + tmpidx_ver_array[inc_idx_num].version = config_seq; + inc_idx_num++; + if(config_seq > latest_inc_version) + { + latest_inc_version = config_seq; + } + } + } + else + { + MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "config file %s,not full or inc config", namelist[i]->d_name); + } + } + + //full update + if(latest_ful_version > current_version) + { + tmpidx_path_array = (struct index_path_array *)malloc(sizeof(struct index_path_array)); + snprintf(tmpidx_path_array->path, MAX_CONFIG_FN_LEN, "%s/%s", file_dir, namelist[full_file_idx]->d_name); + tmpidx_path_array->version = latest_ful_version; + *idx_num = 1; + update_type = CFG_UPDATE_TYPE_FULL; + } + //inc update,it's possible that do inc after full update in this function,but we'll process it at next loop. + else if(latest_inc_version > current_version) + { + tmpidx_path_array = (struct index_path_array *)malloc(sizeof(struct index_path_array)*inc_idx_num); + for(i=0;id_name); + tmpidx_path_array[i].version = tmpidx_ver_array[i].version; + } + *idx_num = inc_idx_num; + update_type = CFG_UPDATE_TYPE_INC; + } + *idx_path = tmpidx_path_array; + + free(tmpidx_ver_array); + for(i=0;icfg_path, "r")) == NULL) + { + MESA_RUNTIME_LOGV4(logger,RLOG_LV_FATAL, "fopen table file %s failed: %s", table->cfg_path, strerror(errno)); + return false; + } + + doris_cbs->cfgfile_start(NULL, table->table_name, table->filesize, table->cfg_num, doris_cbs->userdata); + remainlen = table->filesize; + while(remainlen > 0) + { + oncesize = (remainlen >= ONCE_BUF_SIZE)?ONCE_BUF_SIZE:remainlen; + readlen = fread(scanner->oncebuf, 1, oncesize, fp); + assert(readlen == oncesize); + remainlen -= readlen; + doris_cbs->cfgfile_update(NULL, scanner->oncebuf, readlen, doris_cbs->userdata); + } + doris_cbs->cfgfile_finish(NULL, doris_cbs->userdata); + fclose(fp); + return true; +} + +cJSON *doris_index_version_start(int64_t version, struct cfg_table_info *table_array, int table_num, + enum DORIS_UPDATE_TYPE update_type, struct doris_callbacks *doris_cbs) +{ + cJSON *meta, *array, *tmp; + + meta = cJSON_CreateObject(); + cJSON_AddNumberToObject(meta, "type", update_type); + cJSON_AddNumberToObject(meta, "version", version); + + array = cJSON_CreateArray(); + for(int i=0; iversion_start(NULL, meta, doris_cbs->userdata); + return meta; +} + +enum DORIS_UPDATE_TYPE doris_index_file_traverse(struct doris_idxfile_scanner *scanner, const char*idx_dir, + struct doris_callbacks *doris_cbs, const char* dec_key, void* logger) +{ + enum DORIS_UPDATE_TYPE update_type; + struct index_path_array *idx_path_array=NULL; + int idx_num=0,table_num=0,i=0,j=0; + struct cfg_table_info table_array[CM_MAX_TABLE_NUM]; + bool update_rslt; + + memset(table_array,0,sizeof(table_array)); + update_type=get_new_idx_path(scanner->cur_version, idx_dir,logger, &idx_path_array, &idx_num); + if(update_type == CFG_UPDATE_TYPE_NONE) + { + MESA_RUNTIME_LOGV4(logger,RLOG_LV_DEBUG, "Scan over, no new configs found, waiting next.", idx_path_array[i].path); + return update_type; + } + + update_rslt = true; + for(i=0; icur_version = idx_path_array[i].version; + doris_cbs->version_finish(NULL, doris_cbs->userdata); + } + else + { + update_type = CFG_UPDATE_TYPE_ERR; + doris_cbs->version_error(NULL, doris_cbs->userdata); + cJSON_Delete(meta); + break; + } + cJSON_Delete(meta); + } + free(idx_path_array); + return update_type; +} + +struct doris_idxfile_scanner *doris_index_file_scanner(int64_t start_version) +{ + struct doris_idxfile_scanner *scanner; + + scanner = (struct doris_idxfile_scanner *)malloc(sizeof(struct doris_idxfile_scanner)); + scanner->cur_version = start_version; + return scanner; +} + diff --git a/server/doris_server_scandir.h b/server/doris_server_scandir.h new file mode 100644 index 0000000..fb3fc2c --- /dev/null +++ b/server/doris_server_scandir.h @@ -0,0 +1,52 @@ +#ifndef __DORIS_SERVER_SCANDIR_H__ +#define __DORIS_SERVER_SCANDIR_H__ + +#include "doris_client.h" + +#define CM_MAX_TABLE_NUM 256 +#define MAX_CONFIG_FN_LEN 256 +#define MAX_CONFIG_LINE (1024*16) + +#define ONCE_BUF_SIZE 1048576 + +enum DORIS_UPDATE_TYPE +{ + CFG_UPDATE_TYPE_NONE=0, + CFG_UPDATE_TYPE_FULL=1, + CFG_UPDATE_TYPE_INC=2, + CFG_UPDATE_TYPE_ERR=-1, +}; + +struct index_version_array +{ + int32_t file_inx; + int64_t version; +}; + +struct index_path_array +{ + char path[MAX_CONFIG_FN_LEN]; + int64_t version; +}; + +struct cfg_table_info +{ + char table_name[MAX_CONFIG_FN_LEN]; + char cfg_path[MAX_CONFIG_FN_LEN]; + int cfg_num; + size_t filesize; + char encryp_algorithm[MAX_CONFIG_FN_LEN]; +}; + +struct doris_idxfile_scanner +{ + int64_t cur_version; + char oncebuf[ONCE_BUF_SIZE]; +}; + +struct doris_idxfile_scanner *doris_index_file_scanner(int64_t start_version); +enum DORIS_UPDATE_TYPE doris_index_file_traverse(struct doris_idxfile_scanner *scanner, const char*idx_dir, + struct doris_callbacks *doris_cbs, const char* dec_key, void* logger); + +#endif + diff --git a/support/CMakeLists.txt b/support/CMakeLists.txt new file mode 100644 index 0000000..7247c49 --- /dev/null +++ b/support/CMakeLists.txt @@ -0,0 +1,163 @@ +# CMakeFiles for 3rd vendor library + +include(ExternalProject) + +### OpenSSL 1.1.1 +ExternalProject_Add(OpenSSL PREFIX openssl + URL ${CMAKE_CURRENT_SOURCE_DIR}/openssl-1.1.1.tar.gz + URL_MD5 d65944e4aa4de6ad9858e02c82d85183 + CONFIGURE_COMMAND ./Configure linux-x86_64 --prefix= --openssldir=/lib/ssl + no-weak-ssl-ciphers enable-ec_nistp_64_gcc_128 shared + BUILD_COMMAND ${MAKE_COMMAND} + INSTALL_COMMAND make install_sw + BUILD_IN_SOURCE 1) + +ExternalProject_Get_Property(OpenSSL INSTALL_DIR) +set(OPENSSL_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) +set(OPENSSL_LINK_DIRECTORIES ${INSTALL_DIR}/lib) +set(OPENSSL_PKGCONFIG_PATH ${INSTALL_DIR}/lib/pkgconfig/) +set(OPENSSL_INSTALLED_PATH ${INSTALL_DIR}) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(openssl-crypto-static STATIC IMPORTED GLOBAL) +add_dependencies(openssl-crypto-static OpenSSL) +set_property(TARGET openssl-crypto-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libcrypto.a) +set_property(TARGET openssl-crypto-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +add_library(openssl-ssl-static STATIC IMPORTED GLOBAL) +add_dependencies(openssl-ssl-static OpenSSL) +set_property(TARGET openssl-ssl-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libssl.a) +set_property(TARGET openssl-ssl-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +add_library(openssl-crypto-dynamic SHARED IMPORTED GLOBAL) +add_dependencies(openssl-crypto-dynamic OpenSSL) +set_property(TARGET openssl-crypto-dynamic PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libcrypto.so) +set_property(TARGET openssl-crypto-dynamic PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +add_library(openssl-ssl-dynamic SHARED IMPORTED GLOBAL) +add_dependencies(openssl-ssl-dynamic OpenSSL) +set_property(TARGET openssl-ssl-dynamic PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libssl.so) +set_property(TARGET openssl-ssl-dynamic PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + + +### Libevent 2.1.8 +### Use openssl 1.1.1 +ExternalProject_Add(libevent PREFIX libevent + URL ${CMAKE_CURRENT_SOURCE_DIR}/libevent-2.1.8-stable.tar.gz + URL_MD5 bf4d71806cb1cbc797a39625dbdd721f + CONFIGURE_COMMAND PKG_CONFIG_PATH=${OPENSSL_PKGCONFIG_PATH} + ./configure --prefix= --disable-samples CFLAGS=-fPIC + BUILD_COMMAND make LDFLAGS="-ldl" + BUILD_IN_SOURCE 1 + DEPENDS OpenSSL) + +ExternalProject_Get_Property(libevent INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(libevent-dynamic SHARED IMPORTED GLOBAL) +add_dependencies(libevent-dynamic libevent) +set_property(TARGET libevent-dynamic PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libevent.so) +set_property(TARGET libevent-dynamic PROPERTY IMPORTED_INTERFACE_LINK_LIBRARIES pthread crypto) +set_property(TARGET libevent-dynamic PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +add_library(libevent-static STATIC IMPORTED GLOBAL) +add_dependencies(libevent-static libevent) +set_property(TARGET libevent-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libevent.a) +set_property(TARGET libevent-static PROPERTY IMPORTED_INTERFACE_LINK_LIBRARIES pthread crypto) +set_property(TARGET libevent-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +add_library(libevent-pthreads-static STATIC IMPORTED GLOBAL) +add_dependencies(libevent-pthreads-static libevent) +set_property(TARGET libevent-pthreads-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libevent_pthreads.a) +set_property(TARGET libevent-pthreads-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +add_library(libevent-openssl-dynamic SHARED IMPORTED GLOBAL) +add_dependencies(libevent-openssl-dynamic libevent) +set_property(TARGET libevent-openssl-dynamic PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libevent_openssl.so) +set_property(TARGET libevent-openssl-dynamic PROPERTY IMPORTED_INTERFACE_LINK_LIBRARIES pthread crypto) +set_property(TARGET libevent-openssl-dynamic PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +add_library(libevent-openssl-static STATIC IMPORTED GLOBAL) +add_dependencies(libevent-openssl-static libevent) +set_property(TARGET libevent-openssl-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libevent_openssl.a) +set_property(TARGET libevent-openssl-static PROPERTY IMPORTED_INTERFACE_LINK_LIBRARIES pthread crypto) +set_property(TARGET libevent-openssl-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +### libcurl +ExternalProject_Add(libcurl + PREFIX libcurl + URL ${CMAKE_CURRENT_SOURCE_DIR}/curl-7.71.1.tar.gz + URL_MD5 39bc8d8ae8d59587e863abab739ce7b4 + CONFIGURE_COMMAND ./configure --prefix= --with-ssl=${OPENSSL_INSTALLED_PATH} --disable-ldap --disable-ldaps CFLAGS=-fPIC + BUILD_IN_SOURCE 1 + DEPENDS OpenSSL) + +ExternalProject_Get_Property(libcurl INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(libcurl-static STATIC IMPORTED GLOBAL) +add_dependencies(libcurl-static libcurl) +set_property(TARGET libcurl-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libcurl.a) +set_property(TARGET libcurl-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +add_library(libcurl-dynamic SHARED IMPORTED GLOBAL) +add_dependencies(libcurl-dynamic libcurl) +set_property(TARGET libcurl-dynamic PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libcurl.so) +set_property(TARGET libcurl-dynamic PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + + +### cJSON +ExternalProject_Add(cJSON PREFIX cJSON + URL ${CMAKE_CURRENT_SOURCE_DIR}/cJSON-1.7.7.tar.gz + URL_MD5 9a9d914540ad2cae2114296c623127e6 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DENABLE_LOCALES=Off + -DBUILD_SHARED_AND_STATIC_LIBS=1) + +ExternalProject_Get_Property(cJSON INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(cjson SHARED IMPORTED GLOBAL) +add_dependencies(cjson cJSON) +set_property(TARGET cjson PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib64/libcjson.a) +set_property(TARGET cjson PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) +INSTALL (DIRECTORY ${INSTALL_DIR}/include/ DESTINATION include) + +add_library(cjson-dynamic SHARED IMPORTED GLOBAL) +add_dependencies(cjson-dynamic cJSON) +set_property(TARGET cjson-dynamic PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib64/libcjson.so) +set_property(TARGET cjson-dynamic PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +### MESA Monitor +ExternalProject_Add(MesaMonitor PREFIX MesaMonitor + URL ${CMAKE_CURRENT_SOURCE_DIR}/libmesa_monitor-master.tar.gz + URL_MD5 91fa5666594a4db41def53c461652f19 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DBUILD_SHARED_AND_STATIC_LIBS=1) + +ExternalProject_Get_Property(MesaMonitor INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(libMesaMonitor STATIC IMPORTED GLOBAL) +add_dependencies(libMesaMonitor MesaMonitor) +set_property(TARGET libMesaMonitor PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libMESA_Monitor.a) +set_property(TARGET libMesaMonitor PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + + +#### leveldb +#ExternalProject_Add(LevelDB PREFIX leveldb +# URL ${CMAKE_CURRENT_SOURCE_DIR}/leveldb-1.22.tar.gz +# URL_MD5 ada425fbd00dc0d3d892774bf71f6692 +# CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= +# -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} +# -DBUILD_SHARED_AND_STATIC_LIBS=1) +# +#ExternalProject_Get_Property(LevelDB INSTALL_DIR) +#file(MAKE_DIRECTORY ${INSTALL_DIR}/include) +# +#add_library(libLevelDB STATIC IMPORTED GLOBAL) +#add_dependencies(libLevelDB LevelDB) +#set_property(TARGET libLevelDB PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib64/libleveldb.a) +#set_property(TARGET libLevelDB PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + diff --git a/support/cJSON-1.7.7.tar.gz b/support/cJSON-1.7.7.tar.gz new file mode 100644 index 0000000..5609bfb Binary files /dev/null and b/support/cJSON-1.7.7.tar.gz differ diff --git a/support/curl-7.71.1.tar.gz b/support/curl-7.71.1.tar.gz new file mode 100644 index 0000000..4b7de05 Binary files /dev/null and b/support/curl-7.71.1.tar.gz differ diff --git a/support/libevent-2.1.8-stable.tar.gz b/support/libevent-2.1.8-stable.tar.gz new file mode 100644 index 0000000..f230ea5 Binary files /dev/null and b/support/libevent-2.1.8-stable.tar.gz differ diff --git a/support/libmesa_monitor-master.tar.gz b/support/libmesa_monitor-master.tar.gz new file mode 100644 index 0000000..8724c6a Binary files /dev/null and b/support/libmesa_monitor-master.tar.gz differ diff --git a/support/openssl-1.1.1.tar.gz b/support/openssl-1.1.1.tar.gz new file mode 100644 index 0000000..ceeca12 Binary files /dev/null and b/support/openssl-1.1.1.tar.gz differ