创建

client/CMakeLists.txt

@@ -0,0 +1,31 @@
+set (DORIS_CLIENT_SRC doris_client_fetch.cpp doris_client_http.cpp  doris_client_transfer.cpp  nirvana_conhash.cpp nirvana_murmurhash.cpp)
+
+add_definitions(-fPIC -Wall -g)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_CURRENT_SOURCE_DIR}/,,$(abspath $<))\"'")
+
+add_library(doris_client_static STATIC ${DORIS_CLIENT_SRC})
+set_target_properties(doris_client_static PROPERTIES LINKER_LANGUAGE CXX)
+set_target_properties(doris_client_static PROPERTIES OUTPUT_NAME doris_client)
+set_target_properties(doris_client_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
+target_link_libraries(doris_client_static libevent-static libcurl-static libevent-openssl-static openssl-ssl-static openssl-crypto-static cjson)
+target_link_libraries(doris_client_static MESA_handle_logger MESA_prof_load MESA_field_stat2 pthread)
+target_include_directories(doris_client_static PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_include_directories(doris_client_static PUBLIC ${PROJECT_SOURCE_DIR}/include)
+set_property(TARGET doris_client_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/include)
+
+add_library(doris_client_dynamic SHARED ${DORIS_CLIENT_SRC})
+set_target_properties(doris_client_dynamic PROPERTIES LINKER_LANGUAGE CXX)
+set_target_properties(doris_client_dynamic PROPERTIES OUTPUT_NAME doris_client)
+set_target_properties(doris_client_dynamic PROPERTIES CLEAN_DIRECT_OUTPUT 1)
+target_link_libraries(doris_client_dynamic libevent-dynamic libcurl-dynamic libevent-openssl-dynamic openssl-ssl-dynamic openssl-crypto-dynamic cjson-dynamic)
+target_link_libraries(doris_client_dynamic MESA_handle_logger MESA_prof_load MESA_field_stat2 pthread)
+target_include_directories(doris_client_dynamic PUBLIC ${PROJECT_SOURCE_DIR}/include)
+target_include_directories(doris_client_dynamic PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+set_property(TARGET doris_client_dynamic PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/include)
+
+#INSTALL (TARGETS doris_client_static doris_client_dynamic
+#	LIBRARY DESTINATION lib
+#	ARCHIVE DESTINATION lib)
+#INSTALL (DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION include)
+#INSTALL (FILES doris_client_threads.h doris_conhash.h doris_murmurhash.h DESTINATION include)

client/doris_client_fetch.cpp

@@ -0,0 +1,589 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h> 
+#include <stdio.h> 
+#include <stdlib.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/time.h>
+#include <time.h>
+#include <string.h>
+#include <openssl/sha.h>
+#include <openssl/md5.h>
+
+#include <MESA/MESA_prof_load.h>
+
+#include "doris_client_fetch.h"
+
+void easy_string_destroy(struct easy_string *estr)
+{
+	if(estr->buff != NULL)
+	{
+		free(estr->buff);
+		estr->buff = NULL;
+ 		estr->len = estr->size = 0;
+	}
+}
+
+void easy_string_savedata(struct easy_string *estr, const char *data, size_t len)
+{
+	if(estr->size-estr->len < len+1)
+	{
+		estr->size += len*4+1;
+		estr->buff = (char*)realloc(estr->buff, estr->size);
+	}
+	
+	memcpy(estr->buff+estr->len, data, len);
+	estr->len += len;
+	estr->buff[estr->len]='\0';
+}
+
+void doris_confile_ctx_reset(struct doris_confile_ctx *ctx)
+{
+	struct doris_http_ctx *httpctx=ctx->httpctx;
+	memset(ctx, 0, sizeof(struct doris_confile_ctx));
+	ctx->httpctx = httpctx;
+}
+
+void doris_confile_ctx_destry(struct doris_confile_ctx *ctx)
+{
+	doris_confile_ctx_reset(ctx);
+	doris_http_ctx_destroy(ctx->httpctx);
+	ctx->httpctx = NULL;
+}
+
+void doris_update_new_version(struct doris_instance *instance)
+{
+	instance->cur_version = instance->new_version;
+	instance->new_version += 1;
+}
+
+void doris_request_restart_timer(struct doris_instance *instance, time_t wait_s)
+{
+	struct timeval tv;
+
+	tv.tv_sec = wait_s;
+	tv.tv_usec = 0;
+	event_add(&instance->timer_fetch, &tv);
+}
+
+void doris_fetch_next_confile_meta(struct doris_instance *instance)
+{
+	cJSON *cur_a_item, *sub;
+
+	memset(&instance->curmeta, 0, sizeof(struct fetch_file_meta));
+
+	cur_a_item = cJSON_GetArrayItem(instance->array, instance->array_index);
+	instance->array_index++;
+
+	sub = cJSON_GetObjectItem(cur_a_item, "tablename");
+	instance->curmeta.table_name = sub->valuestring;
+
+	sub = cJSON_GetObjectItem(cur_a_item, "size");
+	instance->curmeta.size = sub->valuedouble;
+
+	sub = cJSON_GetObjectItem(cur_a_item, "cfg_num");
+	instance->curmeta.cfg_num = sub->valueint;
+}
+
+void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code, long res_code, void *userp)
+{
+	struct doris_instance *instance = (struct doris_instance *)userp;
+	const char *pos_colon;
+	size_t datalen;
+	char buffer[64];
+	int ret;
+
+	if(instance->ctx.res_code == 0) //check code only once
+	{
+		instance->ctx.res_code = res_code;
+		assert(res_code != 0);
+
+		if(res_code != 200 && res_code!=206)
+		{
+			MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "Fetch confile %s failed, req_version=%lu, curlcode = %d",
+				instance->curmeta.table_name, instance->new_version, code);
+			return;
+		}
+		instance->retry_times = 0;
+		if(instance->curmeta.curoffset == 0)
+		{
+			instance->param->cbs.cfgfile_start(instance, instance->curmeta.table_name,
+				instance->curmeta.size, instance->curmeta.cfg_num, instance->param->cbs.userdata);
+		}
+	}
+
+	if((pos_colon=(const char*)memchr(start, ':', bytes)) == NULL)
+	{
+		return ;
+	}
+	datalen = pos_colon - start;
+	switch(datalen)
+	{
+		case 14: 
+			if(!strncasecmp(start, "Content-Length:", 15))
+			{
+				memcpy(buffer, start+15, bytes-15);
+				buffer[bytes-15] = '\0';
+				instance->ctx.contlength = atol(buffer);
+			}
+			break;
+		case 13:
+			if(!strncasecmp(start, "Content-Range:", 14))
+			{
+				memcpy(buffer, start+13, bytes-13);
+				buffer[bytes-13] = '\0';
+				ret = sscanf(buffer, "%*[^0-9]%lu-%lu/%lu", &instance->ctx.contl_start, &instance->ctx.contl_end, &instance->ctx.contl_total);
+				assert(ret == 3 && instance->ctx.contl_total == instance->curmeta.size && instance->ctx.contl_start==instance->curmeta.curoffset);
+			}
+			break;
+		default: break;
+	}
+}
+
+void doris_http_confile_body_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp)
+{
+	struct doris_instance *instance = (struct doris_instance *)userp;
+
+	if(code!=CURLE_OK || (instance->ctx.res_code!=200 && instance->ctx.res_code!=206) || (res_code!=200 && res_code!=206))
+	{
+		return;
+	}
+
+	instance->param->cbs.cfgfile_update(instance, ptr, bytes, instance->param->cbs.userdata);
+	instance->curmeta.curoffset += bytes;
+	instance->statistic.field[DRS_FS_FILED_RES_BYTES] += bytes;
+}
+
+void doris_http_fetch_confile(struct doris_instance *instance);
+void doris_http_confile_done_cb(CURLcode res, long res_code, const char *err, void *userp)
+{
+	struct doris_instance *instance = (struct doris_instance *)userp;
+
+	if(instance->ctx.res_code != 200 && instance->ctx.res_code!=206)
+	{
+		goto out_error;
+	}
+
+	if(res!=CURLE_OK || (res_code!=200 && res_code!=206))
+	{
+		MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Fetch confile %s failed, req_version=%lu, curlcode = %d, error: %s",
+			instance->curmeta.table_name, instance->new_version, res_code, err);
+		goto out_error;
+	}
+	if(instance->ctx.contl_total != 0)
+	{
+		MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Fetch confile %s success, req_version=%lu, Content-Range: %lu-%lu/%lu",
+			instance->curmeta.table_name, instance->new_version, instance->ctx.contl_start, instance->ctx.contl_end, instance->ctx.contl_total);
+	}
+	else
+	{
+		MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Fetch confile %s success, req_version=%lu, Content-Length: %lu/%lu",
+			instance->curmeta.table_name, instance->new_version, instance->ctx.contlength, instance->curmeta.size);
+	}
+
+	instance->statistic.field[DRS_FS_FILED_RES_FRAGS] += 1;
+	if(instance->curmeta.curoffset >= instance->curmeta.size) //<2F><><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+	{
+		instance->statistic.field[DRS_FS_FILED_RES_FILES] += 1;
+		instance->param->cbs.cfgfile_finish(instance, instance->param->cbs.userdata);
+		if(instance->array_index == instance->array_size)
+		{
+			instance->param->cbs.version_finish(instance, instance->param->cbs.userdata);
+			instance->status = FETCH_STATUS_META;
+			doris_update_new_version(instance);
+			cJSON_Delete(instance->meta);
+			doris_confile_ctx_destry(&instance->ctx);
+			doris_request_restart_timer(instance, 0);
+		}
+		else
+		{
+			doris_fetch_next_confile_meta(instance);
+			doris_http_fetch_confile(instance);
+		}
+	}
+	else
+	{
+		doris_http_fetch_confile(instance);
+	}
+	return;
+
+out_error:
+	instance->statistic.field[DRS_FS_FILED_RES_FRAGERR] += 1;
+	if(instance->ctx.res_code == 404) //404Ӧ<34><D3A6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>¿<EFBFBD>ʼ<EFBFBD><CABC><EFBFBD><EFBFBD>
+	{
+		instance->retry_times = instance->param->fetch_max_tries;
+	}
+	else
+	{
+		instance->retry_times++;
+	}
+	if(instance->retry_times >= instance->param->fetch_max_tries)
+	{
+		instance->statistic.field[DRS_FS_FILED_RES_VERERR] += 1;
+		instance->param->cbs.version_error(instance, instance->param->cbs.userdata);
+		instance->retry_times = 0;
+		instance->status = FETCH_STATUS_META;
+		cJSON_Delete(instance->meta);
+		doris_confile_ctx_destry(&instance->ctx);
+	}
+	doris_request_restart_timer(instance, instance->param->retry_interval);
+}
+
+void doris_http_fetch_confile(struct doris_instance *instance)
+{
+	struct doris_http_callback curlcbs;
+	char metauri[128], range[64]={0};
+
+	curlcbs.header_cb = doris_http_confile_header_cb;
+	curlcbs.write_cb  = doris_http_confile_body_cb;
+	curlcbs.transfer_done_cb = doris_http_confile_done_cb;
+	curlcbs.userp = instance;
+	doris_confile_ctx_reset(&instance->ctx);
+	doris_http_ctx_reset(instance->ctx.httpctx, &curlcbs);
+
+	//<2F><><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC>ֶ<EFBFBD><D6B6><EFBFBD><EFBFBD>أ<EFBFBD><D8A3>ϴ<EFBFBD>δ<EFBFBD><CEB4><EFBFBD>ɵļ<C9B5><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+	if((instance->curmeta.size > instance->param->fetch_frag_size) || instance->curmeta.curoffset!=0)
+	{
+		sprintf(range, "Range: bytes=%lu-%lu", instance->curmeta.curoffset, instance->curmeta.curoffset + instance->param->fetch_frag_size - 1);
+		doris_http_ctx_add_header(instance->ctx.httpctx, range);
+	}
+
+	snprintf(metauri, 128, "configfile?tablename=%s&version=%lu&businessid=%u", instance->curmeta.table_name, instance->new_version, instance->param->args.businessid);
+	if(doris_http_launch_get_request(instance->ctx.httpctx, metauri))
+	{
+		instance->statistic.field[DRS_FS_FILED_REQ_FAIL] += 1;
+		doris_request_restart_timer(instance, instance->param->retry_interval);
+	}
+	else
+	{
+		instance->statistic.field[DRS_FS_FILED_REQ_FILES] += 1;
+		MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Launch confile %s GET, req_version=%lu, %s",
+			instance->curmeta.table_name, instance->new_version, range);
+	}
+}
+
+void doris_http_meta_header_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp)
+{
+	struct doris_instance *instance = (struct doris_instance *)userp;
+
+	//check code only once
+	if(instance->ctx.res_code != 0)
+	{
+		return;
+	}
+	instance->ctx.res_code = res_code;
+	assert(res_code != 0);
+
+	if(res_code != 200)
+	{
+		MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "No new meta found, cur_version=%lu, req_version=%lu, curlcode = %d",
+			instance->cur_version, instance->new_version, code);
+	}
+}
+
+void doris_http_meta_body_cb(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp)
+{
+	struct doris_instance *instance = (struct doris_instance *)userp;
+
+	if(code!=CURLE_OK || res_code!=200)
+	{
+		return;
+	}
+
+	easy_string_savedata(&instance->estr, (const char*)ptr, bytes);
+}
+
+void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void *userp)
+{
+	struct doris_instance *instance = (struct doris_instance *)userp;
+
+	if(instance->ctx.res_code != 200)
+	{
+		goto out_error;
+	}
+
+	if(res!=CURLE_OK || res_code!=200)
+	{
+		MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "No new meta found, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s",
+			instance->cur_version, instance->new_version, res_code, err);
+		goto out_error;
+	}
+
+	instance->meta = cJSON_Parse(instance->estr.buff);
+	if(instance->meta == NULL)
+	{
+		MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Parse meta failed, req_version=%lu, invalid json: %s", instance->new_version, instance->estr.buff);
+		goto out_error;
+	}
+	instance->statistic.field[DRS_FS_FILED_RES_META] += 1;
+	MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, cur_version=%lu, newjson: %s",
+		instance->cur_version, instance->estr.buff);
+
+	instance->param->cbs.version_start(instance, instance->meta, instance->param->cbs.userdata);
+	instance->array = cJSON_GetObjectItem(instance->meta, "configs");
+	instance->array_size = cJSON_GetArraySize(instance->array);
+	assert(instance->array_size > 0);
+
+	easy_string_destroy(&instance->estr);
+	instance->status = FETCH_STATUS_FILE;
+	doris_fetch_next_confile_meta(instance);
+	doris_http_fetch_confile(instance);
+	return;
+
+out_error:
+	instance->statistic.field[DRS_FS_FILED_RES_NOMETA] += 1;
+	doris_request_restart_timer(instance, instance->param->retry_interval);
+	easy_string_destroy(&instance->estr);
+	doris_confile_ctx_destry(&instance->ctx);
+}
+
+static void doris_http_fetch_meta(struct doris_instance *instance)
+{
+	u_int64_t balance_seed;
+	struct doris_http_callback curlcbs;
+	char metauri[128];
+
+	balance_seed = (((u_int64_t)rand()) << 32) | rand();
+
+	memset(&curlcbs, 0, sizeof(struct doris_http_callback));
+	curlcbs.header_cb = doris_http_meta_header_cb;
+	curlcbs.write_cb  = doris_http_meta_body_cb;
+	curlcbs.transfer_done_cb = doris_http_meta_done_cb;
+	curlcbs.userp = instance;
+
+	instance->array_index = 0;
+	instance->cur_httpins = instance->httpins_master;
+	instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
+	if(instance->ctx.httpctx==NULL && instance->httpins_backup1!=NULL)
+	{
+		instance->cur_httpins = instance->httpins_backup1;
+		instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
+	}
+	if(instance->ctx.httpctx==NULL && instance->httpins_backup2!=NULL)
+	{
+		instance->cur_httpins = instance->httpins_backup2;
+		instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
+	}
+
+	if(instance->ctx.httpctx != NULL)
+	{
+		snprintf(metauri, 128, "configmeta?version=%lu&businessid=%u", instance->new_version, instance->param->args.businessid);
+		if(!doris_http_launch_get_request(instance->ctx.httpctx, metauri))
+		{
+			instance->status = FETCH_STATUS_META;
+			instance->statistic.field[DRS_FS_FILED_REQ_META] += 1;
+		}
+		else
+		{
+			instance->statistic.field[DRS_FS_FILED_REQ_FAIL] += 1;
+			doris_confile_ctx_destry(&instance->ctx);
+			doris_request_restart_timer(instance, instance->param->retry_interval);
+		}
+		if(instance->cur_httpins == instance->httpins_backup1) instance->statistic.field[DRS_FS_FILED_BACKUP1_REQ] += 1;
+		else if(instance->cur_httpins == instance->httpins_backup2) instance->statistic.field[DRS_FS_FILED_BACKUP2_REQ] += 1;
+	}
+	else
+	{
+		instance->statistic.field[DRS_FS_FILED_REQ_FAIL] += 1;
+		doris_request_restart_timer(instance, instance->param->retry_interval);
+		MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Launch meta GET failed: no active host found,req_version=%lu", instance->new_version);
+	}	
+}
+
+static void instance_fetch_cfg_timer_cb(int fd, short kind, void *userp)
+{
+	struct doris_instance *instance = (struct doris_instance *)userp;
+
+	switch(instance->status)
+	{
+		case FETCH_STATUS_IDLE:
+		case FETCH_STATUS_META:
+			doris_http_fetch_meta(instance);
+			break;
+
+		case FETCH_STATUS_FILE:
+			doris_http_fetch_confile(instance);
+			break;
+		default: assert(0);break;
+	}
+}
+
+static void doris_client_fs_output_timer_cb(int fd, short kind, void *userp)
+{
+	struct doris_parameter *param=(struct doris_parameter *)userp;
+	struct timeval tv;
+
+	FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_MST_CNN_SRV], 0, FS_OP_SET, param->param_master->connected_hosts);
+	FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_MST_FAIL_SRV], 0, FS_OP_SET, param->param_master->failed_hosts);
+	if(param->param_backup1 != NULL)
+	{
+		FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_BCK1_CNN_SRV], 0, FS_OP_SET, param->param_backup1->connected_hosts);
+		FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_BCK1_FAIL_SRV], 0, FS_OP_SET, param->param_backup1->failed_hosts);
+	}
+	if(param->param_backup2 != NULL)
+	{
+		FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_BCK2_CNN_SRV], 0, FS_OP_SET, param->param_backup2->connected_hosts);
+		FS_operate(param->fsstat_handle, param->fsstat_status[DRS_FS_STAT_BCK2_FAIL_SRV], 0, FS_OP_SET, param->param_backup2->failed_hosts);
+	}
+	FS_passive_output(param->fsstat_handle);
+	tv.tv_sec = param->fsstat_period;
+	tv.tv_usec = 0;
+	evtimer_add(&param->fs_timer_output, &tv);
+}
+
+static int doris_client_register_field_stat(struct doris_parameter *param, void *runtime_log, struct event_base *evbase)
+{
+	const char *field_names[FSSTAT_DORIS_FILED_NUM]={"ReqFail", "ReqMetas", "ResMetas", "ResNoNew", "ReqFiles",
+		"ResFiles", "ResFrags", "ResFragErr", "ResBytes", "ResVerErr", "ReqBackup1", "ReqBackup2"};
+	const char *status_names[FSSTAT_DORIS_STATUS_NUM]={"MasSrvCned", "MasSrvFail",
+		"Bck1SrvCned", "Bck1SrvFail", "Bck2SrvCned", "Bck2SrvFail", "MemoryUsed", "HttpSession"};
+	struct timeval tv;
+	int value;
+
+	param->fsstat_handle = FS_create_handle();
+	FS_set_para(param->fsstat_handle, OUTPUT_DEVICE, param->fsstat_filepath, strlen(param->fsstat_filepath)+1);
+	if(param->fsstat_print_mode == 1)
+	{
+		FS_set_para(param->fsstat_handle, PRINT_MODE, &param->fsstat_print_mode, sizeof(param->fsstat_print_mode));
+	}
+	else
+	{
+		FS_set_para(param->fsstat_handle, PRINT_MODE, &param->fsstat_print_mode, sizeof(param->fsstat_print_mode));
+		value = 1;
+		FS_set_para(param->fsstat_handle, FLUSH_BY_DATE, &value, sizeof(value));
+	}
+	value = param->fsstat_period;
+	FS_set_para(param->fsstat_handle, STAT_CYCLE, &value, sizeof(value));
+	value = 0;
+	FS_set_para(param->fsstat_handle, CREATE_THREAD, &value, sizeof(value));
+	FS_set_para(param->fsstat_handle, APP_NAME, param->fsstat_appname, strlen(param->fsstat_appname)+1);
+	FS_set_para(param->fsstat_handle, STATS_SERVER_IP, param->fsstat_dst_ip, strlen(param->fsstat_dst_ip)+1);
+	FS_set_para(param->fsstat_handle, STATS_SERVER_PORT, &param->fsstat_dst_port, sizeof(param->fsstat_dst_port));
+
+	for(int i=0; i<FSSTAT_DORIS_FILED_NUM; i++)
+	{
+		param->fsstat_field[i] = FS_register(param->fsstat_handle, FS_STYLE_FIELD, FS_CALC_CURRENT, field_names[i]);
+	}
+	for(int i=0; i<FSSTAT_DORIS_STATUS_NUM; i++)
+	{
+		param->fsstat_status[i] = FS_register(param->fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, status_names[i]);
+	}
+	FS_start(param->fsstat_handle);
+
+	evtimer_assign(&param->fs_timer_output, evbase, doris_client_fs_output_timer_cb, param);
+	tv.tv_sec = param->fsstat_period;
+	tv.tv_usec = 0;
+	evtimer_add(&param->fs_timer_output, &tv);
+	return 0;
+}
+
+struct doris_parameter *doris_parameter_new(const char *confile, struct event_base *manage_evbase, struct doris_callbacks *cbs,
+	struct doris_arguments *args, void *runtimelog)
+{
+	struct doris_parameter *param;
+
+	param = (struct doris_parameter *)calloc(1, sizeof(struct doris_parameter));
+	param->manage_evbase = manage_evbase;
+	param->cbs = *cbs;
+	param->args= *args;
+
+	MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_fail_retry_interval", &param->retry_interval, 10);
+	MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_fragmet_size", &param->fetch_frag_size, 5242880);
+	MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fetch_confile_max_tries", &param->fetch_max_tries, 3);
+
+	MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_appname", param->fsstat_appname, 16, "DorisClient");
+	MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_filepath", param->fsstat_filepath, 256, "./log/doris_client.fs");
+	MESA_load_profile_uint_def(confile, "DORIS_CLIENT", "fsstat_log_interval", &param->fsstat_period, 10);
+	MESA_load_profile_int_def(confile, "DORIS_CLIENT", "fsstat_log_print_mode", &param->fsstat_print_mode, 1);
+	MESA_load_profile_string_def(confile, "DORIS_CLIENT", "fsstat_log_dst_ip", param->fsstat_dst_ip, 64, "127.0.0.1");
+	MESA_load_profile_int_def(confile, "DORIS_CLIENT", "fsstat_log_dst_port", &param->fsstat_dst_port, 8125);
+
+	param->param_master = doris_http_parameter_new(confile, "DORIS_CLIENT.master_server", manage_evbase, runtimelog);
+	if(param->param_master == NULL)
+	{
+		return NULL;
+	}
+	if(doris_client_register_field_stat(param, runtimelog, manage_evbase))
+	{
+		return NULL;
+	}
+	param->param_backup1 = doris_http_parameter_new(confile, "DORIS_CLIENT.backup1_server", manage_evbase, runtimelog);
+	param->param_backup2 = doris_http_parameter_new(confile, "DORIS_CLIENT.backup2_server", manage_evbase, runtimelog);
+	return param;	
+}
+
+static void doris_instance_statistic_timer_cb(int fd, short kind, void *userp)
+{
+	struct doris_instance *instance = (struct doris_instance *)userp;
+	struct timeval tv;
+	struct doris_statistics incr_statistic;
+	long long *plast_statistic = (long long*)&instance->statistic_last;
+	long long *pnow_statistic = (long long*)&instance->statistic;
+	long long *pinc_statistic = (long long*)&incr_statistic;
+	long long http_sessions=0;
+
+	http_sessions += caculate_http_sessions_sum(instance->httpins_master);
+	http_sessions += caculate_http_sessions_sum(instance->httpins_backup1);
+	http_sessions += caculate_http_sessions_sum(instance->httpins_backup2);
+	instance->statistic.field[DRS_FS_STAT_HTTP_SESSIONS] = http_sessions;
+
+	for(u_int32_t i=0; i<sizeof(struct doris_statistics)/sizeof(long long); i++)
+	{
+		pinc_statistic[i] = pnow_statistic[i] - plast_statistic[i];
+	}
+	instance->statistic_last = instance->statistic;
+
+	for(u_int32_t i=0; i<FSSTAT_DORIS_FILED_NUM; i++)
+	{
+		FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field[i], 0, FS_OP_ADD, incr_statistic.field[i]);
+	}
+	for(u_int32_t i=0; i<FSSTAT_DORIS_STATUS_NUM; i++)
+	{
+		FS_operate(instance->param->fsstat_handle, instance->param->fsstat_status[i], 0, FS_OP_ADD, incr_statistic.status[i]);
+	}
+	tv.tv_sec = instance->param->fsstat_period;
+	tv.tv_usec = 0;
+	event_add(&instance->timer_statistic, &tv);
+}
+
+struct doris_instance *doris_instance_new(struct doris_parameter *param, struct event_base *worker_evbase, void *runtimelog)
+{
+	struct doris_instance *instance;
+	struct timeval tv;
+
+	instance = (struct doris_instance *)calloc(1, sizeof(struct doris_instance));
+	instance->param = param;
+	instance->worker_evbase = worker_evbase;
+	instance->runtime_log = runtimelog;
+	instance->cur_version = param->args.current_version;
+	instance->new_version = instance->cur_version + 1; //TODO
+
+	instance->httpins_master = doris_http_instance_new(param->param_master, worker_evbase, runtimelog);
+	if(instance->httpins_master == NULL)
+	{
+		return NULL;
+	}
+	srand(time(NULL));
+	
+	if(param->param_backup1 != NULL)
+	{
+		instance->httpins_backup1 = doris_http_instance_new(param->param_backup1, worker_evbase, runtimelog);
+	}
+	if(param->param_backup2 != NULL)
+	{
+		instance->httpins_backup2 = doris_http_instance_new(param->param_backup2, worker_evbase, runtimelog);
+	}
+
+	evtimer_assign(&instance->timer_statistic, worker_evbase, doris_instance_statistic_timer_cb, instance);
+	tv.tv_sec = param->fsstat_period;
+	tv.tv_usec = 0;
+	evtimer_add(&instance->timer_statistic, &tv);
+
+	evtimer_assign(&instance->timer_fetch, worker_evbase, instance_fetch_cfg_timer_cb, instance);
+	tv.tv_sec = 3;
+	tv.tv_usec = 0;
+	evtimer_add(&instance->timer_fetch, &tv);
+	return instance;
+}
+

client/doris_client_fetch.h

@@ -0,0 +1,107 @@
+#ifndef __DORIS_CLIENT_FETCH_IN_H__
+#define __DORIS_CLIENT_FETCH_IN_H__
+
+#include <MESA/field_stat2.h>
+
+#include "doris_client.h"
+#include "doris_client_http.h"
+
+struct easy_string
+{
+	char* buff;
+	size_t len;
+	size_t size;
+};
+
+enum FETCH_CFG_STATUS
+{
+	FETCH_STATUS_IDLE=0,
+	FETCH_STATUS_META,
+	FETCH_STATUS_FILE,
+};
+
+struct doris_parameter
+{
+	struct doris_callbacks cbs;
+	struct doris_arguments args;
+
+	u_int32_t retry_interval;
+	u_int32_t fetch_frag_size;
+	u_int32_t fetch_max_tries;
+
+	struct doris_http_parameter *param_master;
+	struct doris_http_parameter *param_backup1;
+	struct doris_http_parameter *param_backup2;
+	struct event_base *manage_evbase;
+
+	screen_stat_handle_t fsstat_handle;
+	struct event fs_timer_output;
+	char fsstat_dst_ip[64];
+	char fsstat_appname[16];
+	char fsstat_filepath[256];
+	u_int32_t fsstat_period;
+	int32_t fsstat_print_mode;
+	int32_t fsstat_dst_port;
+	int32_t fsstat_field[FSSTAT_DORIS_FILED_NUM];
+	int32_t fsstat_status[FSSTAT_DORIS_STATUS_NUM];
+};
+
+struct md5_long
+{
+	u_int64_t md5l;
+	u_int64_t md5h;
+};
+
+struct fetch_file_meta
+{
+	const char *table_name;
+	size_t size;
+	size_t curoffset;
+	u_int32_t cfg_num;
+	union {
+		char md5[16];
+		struct md5_long md5long;
+	};
+};
+
+struct doris_confile_ctx
+{
+	struct doris_http_ctx *httpctx;
+
+	long res_code;
+	size_t contlength;
+	size_t contl_start;
+	size_t contl_end;
+	size_t contl_total;
+};
+
+struct doris_instance
+{
+	enum FETCH_CFG_STATUS status;
+	u_int32_t retry_times;
+
+	struct doris_http_instance *cur_httpins;
+	int64_t cur_version;
+	int64_t new_version;
+	struct easy_string estr;
+	cJSON *meta, *array;
+	u_int32_t array_size;
+	u_int32_t array_index;
+	struct fetch_file_meta curmeta;
+
+	struct doris_confile_ctx ctx;
+
+	struct doris_http_instance *httpins_master;
+	struct doris_http_instance *httpins_backup1;
+	struct doris_http_instance *httpins_backup2;
+
+	struct event_base *worker_evbase;
+	struct event timer_fetch;
+	struct doris_parameter *param;
+	struct event timer_statistic;
+	struct doris_statistics statistic, statistic_last;
+	void *runtime_log;
+};
+
+#endif
+

client/doris_client_http.cpp

@@ -0,0 +1,374 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <net/if.h>
+#include <unistd.h> 
+#include <stdio.h> 
+#include <stdlib.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/time.h>
+#include <time.h>
+#include <string.h>
+#include <openssl/sha.h>
+#include <openssl/md5.h>
+
+#include <MESA/MESA_prof_load.h>
+
+#include "doris_client_http.h"
+
+int32_t param_get_connected_hosts(struct doris_http_parameter *param)
+{
+	return param->connected_hosts;
+}
+int32_t param_get_failed_hosts(struct doris_http_parameter *param)
+{
+	return param->failed_hosts;
+}
+
+static int _unfold_IP_range(char* ip_range, char***ip_list, int size)
+{
+	int i=0,count=0, ret=0;
+	int range_digits[5];
+	memset(range_digits,0,sizeof(range_digits));
+	ret=sscanf(ip_range,"%d.%d.%d.%d-%d",&range_digits[0],&range_digits[1],&range_digits[2],&range_digits[3],&range_digits[4]);
+	if(ret!=4&&ret!=5)
+	{
+		return 0;
+	}
+	if(ret==4&&range_digits[4]==0)
+	{
+		range_digits[4]=range_digits[3];
+	}
+	for(i=0;i<5;i++)
+	{
+		if(range_digits[i]<0||range_digits[i]>255)
+		{
+			return 0;
+		}
+	}
+	count=range_digits[4]-range_digits[3]+1;
+	*ip_list=(char**)realloc(*ip_list, sizeof(char*)*(size+count));
+	for(i=0;i<count;i++)
+	{
+		(*ip_list)[size+i]=(char*)malloc(64);
+		snprintf((*ip_list)[size+i],64,"%d.%d.%d.%d",range_digits[0],range_digits[1],range_digits[2],range_digits[3]+i);
+	}
+	return count;
+}
+
+static int unfold_IP_range(const char* ip_range, char***ip_list)
+{
+	char *token=NULL,*sub_token=NULL,*saveptr;
+	char *buffer=(char*)calloc(sizeof(char),strlen(ip_range)+1);
+	int count=0;
+	strcpy(buffer,ip_range);
+	for (token = buffer; ; token= NULL)
+	{
+		sub_token= strtok_r(token,";", &saveptr);
+		if (sub_token == NULL)
+			break;
+		count+=_unfold_IP_range(sub_token, ip_list,count);	
+	}
+	free(buffer);
+	return count;
+}
+
+static int decode_one_specific_group_ip(const char *iplist, struct dst_ipaddr_group *dstaddr)
+{
+	int i, ipaddr_num;
+	char **balance_iplist=NULL;
+	
+	ipaddr_num = unfold_IP_range(iplist, &balance_iplist);
+	if(ipaddr_num == 0)
+	{
+		free(dstaddr);
+		return -1;
+	}
+	dstaddr->dstaddr_num = ipaddr_num;
+	dstaddr->dstaddrs = (u_int32_t *)calloc(1, sizeof(u_int32_t)*ipaddr_num);
+	for(i=0; i<ipaddr_num; i++)
+	{
+		if(inet_pton(AF_INET, balance_iplist[i], (void *)&dstaddr->dstaddrs[i]) != 1)
+		{
+			free(dstaddr->dstaddrs);
+			return -1;
+		}
+		free(balance_iplist[i]);
+	}
+	free(balance_iplist);
+	return 0;
+}
+
+static int32_t load_and_init_server_group(char *dst_ipaddr, struct dst_ipaddr_group *dstaddrs, void *runtime_log)
+{
+	u_int32_t intval;
+
+	intval = strlen(dst_ipaddr);
+	if(dst_ipaddr[intval-1] != ';')
+	{
+		dst_ipaddr[intval] = ';';
+		dst_ipaddr[intval+1] = '\0';
+	}
+	if(decode_one_specific_group_ip(dst_ipaddr, dstaddrs) < 0)
+	{
+		return -2;
+	}
+	return 0;
+}
+
+static void conhash_delay_destroy_timer_cb(int fd, short kind, void *userp)
+{
+	struct time_event *delay_event=(struct time_event *)userp;
+
+	conhash_instance_free((struct consistent_hash *)delay_event->data);
+	free(delay_event);
+}
+
+static void load_balance_common_timer_start(struct event *time_event)
+{
+	struct timeval tv;
+
+	tv.tv_sec = 2;
+	tv.tv_usec = 0;
+	evtimer_add(time_event, &tv);
+}
+
+static void conhash_handle_delay_destroy(struct event_base *evbase, struct consistent_hash *conhash)
+{
+	struct time_event *delay_event;
+
+	delay_event = (struct time_event *)malloc(sizeof(struct time_event));
+	delay_event->data = conhash;
+	evtimer_assign(&delay_event->timer_event, evbase, conhash_delay_destroy_timer_cb, delay_event);
+	load_balance_common_timer_start(&delay_event->timer_event);
+}
+
+static void conhash_insert_dest_host(struct dst_host_cnn_balance *balance)
+{
+	struct conhash_bucket bucket;
+	struct consistent_hash *tmphash, *newhash=NULL;
+	enum CONHASH_ERRCODE code;
+	
+	bucket.bucket_id = balance->dstip;
+	bucket.point_num = DEFAULT_HOST_CAPACITY * LOAD_BALANC_VIRT_TIMES;;
+	bucket.tag = NULL;
+
+	newhash = conhash_instance_copy(balance->param->conhash);
+	code = conhash_insert_bucket(newhash, &bucket);
+	assert(code == CONHASH_OK);
+	
+	tmphash = balance->param->conhash;
+	balance->param->conhash = newhash;
+	conhash_handle_delay_destroy(balance->param->evbase, tmphash);
+}
+
+static void conhash_remove_dest_host(struct dst_host_cnn_balance *balance)
+{
+	struct consistent_hash *tmphash, *newhash=NULL;
+	enum CONHASH_ERRCODE code;
+	
+	newhash = conhash_instance_copy(balance->param->conhash);
+	code = conhash_remove_bucket(newhash, balance->dstip, NULL);
+	assert(code == CONHASH_OK || code==CONHASH_BUCKET_NOT_FOUND);
+	
+	tmphash = balance->param->conhash;
+	balance->param->conhash = newhash;
+	conhash_handle_delay_destroy(balance->param->evbase, tmphash);
+}
+
+static void client_bufferevent_error_cb(struct bufferevent *bev, short event, void *arg)  
+{
+	struct dst_host_cnn_balance *balance = (struct dst_host_cnn_balance *)arg;
+	const char *errtype;
+	
+	if(event & BEV_EVENT_CONNECTED) 
+	{  
+		errtype = "connected";
+		balance->connection_status = TCP_STATUS_CONNECTED;
+		balance->param->connected_hosts += 1; //<2F><><EFBFBD>й<EFBFBD><D0B9><EFBFBD><EFBFBD>߳<EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>̽<EFBFBD><CCBD>
+		if(balance->connect_failed)
+		{
+			balance->connect_failed = false;
+			balance->param->failed_hosts -= 1;
+		}
+		conhash_insert_dest_host(balance);
+		assert(balance->param->connected_hosts > 0);
+		assert(balance->param->failed_hosts >= 0);
+	}
+	else
+	{
+		if (event & BEV_EVENT_TIMEOUT) {  
+			errtype = "Timed out";  
+		}
+		else if (event & BEV_EVENT_EOF) {  
+			errtype = "disconnected";  
+		}
+		else if (event & BEV_EVENT_ERROR) {
+			errtype = "some other error"; 
+		}
+		else {
+			errtype = "unkonwn error";
+		}
+		bufferevent_free(bev);
+		balance->bev = NULL;
+
+		if(balance->connection_status == TCP_STATUS_CONNECTED)
+		{
+			balance->param->connected_hosts -= 1;
+		}
+		if(!balance->connect_failed)
+		{
+			balance->connect_failed = true;
+			balance->param->failed_hosts += 1;
+		}
+		balance->connection_status = TCP_STATUS_DISCONNECT;
+	
+		load_balance_common_timer_start(&balance->timer_detect);
+		conhash_remove_dest_host(balance);
+
+		assert(balance->param->connected_hosts >= 0);
+		assert(balance->param->failed_hosts > 0);
+	}
+	
+	MESA_HANDLE_RUNTIME_LOGV2(balance->param->runtime_log, RLOG_LV_INFO, "connection event: %s, addr: %s", errtype, balance->srvaddr);
+}
+
+int do_bufferevent_connection(struct doris_http_parameter *param, struct dst_host_cnn_balance *balance, struct sockaddr *server_addr)
+{
+	if(balance->connection_status==TCP_STATUS_CONNECTING)
+	{
+		return 0;
+	}
+
+	if(NULL == (balance->bev = bufferevent_socket_new(param->evbase, -1, BEV_OPT_CLOSE_ON_FREE)))
+	{
+		assert(0);return -1;
+	}
+	
+	if(bufferevent_socket_connect(balance->bev, server_addr, sizeof(struct sockaddr_in)))
+	{
+		bufferevent_free(balance->bev);
+		balance->bev = NULL;
+		MESA_HANDLE_RUNTIME_LOGV2(param->runtime_log, RLOG_LV_FATAL, "bufferevent_socket_connect error: %s", strerror(errno));
+		assert(0);return -2;
+	}
+	balance->connection_status = TCP_STATUS_CONNECTING;
+	bufferevent_setcb(balance->bev, NULL, NULL, client_bufferevent_error_cb, balance);
+	bufferevent_setwatermark(balance->bev, EV_WRITE, 100*1024*1024UL, 0);
+	bufferevent_enable(balance->bev, EV_READ|EV_WRITE|EV_PERSIST);
+	return 0;
+}
+
+void param_connection_detect_timer_cb(int fd, short kind, void *userp)
+{
+	struct dst_host_cnn_balance *balance = (struct dst_host_cnn_balance *)userp;
+
+	if(do_bufferevent_connection(balance->param, balance, (struct sockaddr*)&balance->addr))
+	{
+		assert(0);
+	}
+}
+
+static int32_t doris_launch_group_connection(struct doris_http_parameter *param, struct event_base* evbase)
+{
+	char ipaddr[64];
+
+	param->balance = (struct dst_host_cnn_balance *)calloc(1, sizeof(struct dst_host_cnn_balance)*param->ipgroup.dstaddr_num);
+	
+	for(u_int32_t i=0; i<param->ipgroup.dstaddr_num; i++) //<2F><><EFBFBD><EFBFBD>
+	{
+		param->balance[i].dstip = param->ipgroup.dstaddrs[i];
+		param->balance[i].addr.sin_family = AF_INET;
+		param->balance[i].addr.sin_port = htons(param->manage_port);
+		param->balance[i].addr.sin_addr.s_addr = param->ipgroup.dstaddrs[i];
+		param->balance[i].connection_status = TCP_STATUS_IDLE;
+		param->balance[i].param = param;
+	
+		inet_ntop(AF_INET, &param->ipgroup.dstaddrs[i], ipaddr, 64);
+		snprintf(param->balance[i].srvaddr, 64, "%s:%u", ipaddr, param->server_port);
+
+		evtimer_assign(&param->balance[i].timer_detect, evbase, param_connection_detect_timer_cb, &param->balance[i]);
+	
+		if(do_bufferevent_connection(param, &param->balance[i], (struct sockaddr *)&param->balance[i].addr))
+		{
+			MESA_HANDLE_RUNTIME_LOGV2(param->runtime_log, RLOG_LV_FATAL, "do_bufferevent_connection error: %s.", strerror(errno));
+			return -1;
+		}
+	}
+	return 0;
+}
+
+struct doris_http_parameter *doris_http_parameter_new(const char* profile_path, const char* section, struct event_base* evbase, void *runtime_log)
+{
+	struct doris_http_parameter *param;
+	u_int32_t intval;
+	char dst_ipaddr[8192];
+
+	param = (struct doris_http_parameter *)calloc(1, sizeof(struct doris_http_parameter));
+	param->runtime_log = runtime_log;
+	param->evbase = evbase;
+
+	//multi curl
+	MESA_load_profile_uint_def(profile_path, section, "max_connection_per_host", &intval, 1);
+	param->maximum_host_cnns = intval;
+	MESA_load_profile_uint_def(profile_path, section, "max_cnnt_pipeline_num", &intval, 20);
+	param->maximum_pipelines = intval;
+	MESA_load_profile_uint_def(profile_path, section, "max_curl_transfer_timeout_s", &intval, 0);
+	param->transfer_timeout = intval;
+	MESA_load_profile_int_def(profile_path, section, "https_connection_on", &param->ssl_connection, 0);
+	MESA_load_profile_uint_def(profile_path, section, "max_curl_session_num", &param->max_http_sessions, 30);
+
+	//Server
+	if(MESA_load_profile_uint_def(profile_path, section, "http_server_listen_port", &param->server_port, 9898) < 0)
+	{
+		free(param);
+		MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] http_server_listen_port not found.", profile_path, section);
+		return NULL;
+	}
+	MESA_load_profile_uint_def(profile_path, section, "http_server_manage_port", &param->manage_port, 2233);
+
+	if(MESA_load_profile_string_nodef(profile_path, section, "http_server_ip_list", dst_ipaddr, 8192) < 0)
+	{
+		free(param);
+		MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] http_server_ip_list not found.", profile_path, section);
+		return NULL;
+	}
+	
+	if(load_and_init_server_group(dst_ipaddr, &param->ipgroup, runtime_log))
+	{
+		MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Decode %s [%s] http_server_ip_list format error: %s", profile_path, section, dst_ipaddr);
+		assert(0);return NULL;
+	}
+	param->conhash = conhash_instance_new(NULL, 0);;
+	if(doris_launch_group_connection(param, evbase))
+	{
+		assert(0);return NULL;
+	}
+	return param;
+}
+
+struct doris_http_instance *doris_http_instance_new(struct doris_http_parameter *param, struct event_base* evbase, void *runtimelog)
+{
+	struct doris_http_instance *instance;
+	struct doris_curl_multihd *multihd;
+
+	instance = (struct doris_http_instance *)calloc(1, sizeof(struct doris_http_instance));
+	instance->runtime_log = runtimelog;	
+	instance->evbase = evbase;
+	instance->param = param;
+	
+	//Ϊÿһ<C3BF><D2BB>minio IP<49><50><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>multi_handle
+	instance->server_hosts = new map<u_int32_t, doris_curl_multihd*>;
+	for(u_int32_t i=0; i<param->ipgroup.dstaddr_num; i++)
+	{
+		multihd = doris_initialize_multihd_for_host(instance, &param->balance[i]);
+		instance->server_hosts->insert(make_pair(param->ipgroup.dstaddrs[i], multihd));
+	}
+	return instance;
+}
+

client/doris_client_http.h

@@ -0,0 +1,108 @@
+#ifndef __DORIS_CLIENT_HTTP_IN_H__
+#define __DORIS_CLIENT_HTTP_IN_H__
+
+#include <openssl/ssl.h>
+#include <openssl/crypto.h>
+#include <openssl/x509.h>
+#include <openssl/pem.h>
+#include <openssl/err.h>
+#include <openssl/rsa.h>
+#include <event.h>
+
+#include <MESA/MESA_handle_logger.h>
+
+#include <map>
+#include <string>
+
+#include "doris_client_transfer.h"
+#include "nirvana_conhash.h"
+
+using namespace std;
+
+#ifndef __FILENAME__
+#define __FILENAME__ __FILE__
+#endif
+#define MESA_HANDLE_RUNTIME_LOGV2(handle, lv, fmt, args...)   \
+    MESA_handle_runtime_log((handle), (lv), "DorisClient", "%s:%d, " fmt, __FILENAME__, __LINE__, ##args)
+
+#define DEFAULT_HOST_CAPACITY  4
+#define LOAD_BALANC_VIRT_TIMES 16
+
+enum TCP_CONNECTION_STATUS
+{
+	TCP_STATUS_IDLE=0,
+	TCP_STATUS_CONNECTING,    //ֻ<><D6BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ӲŻỺ<C5BB><E1BBBA><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+	TCP_STATUS_CONNECTED,
+	TCP_STATUS_DISCONNECT,
+};
+
+
+struct dst_ipaddr_group
+{
+	u_int32_t *dstaddrs;
+	u_int32_t dstaddr_num;
+};
+
+struct doris_http_parameter;
+struct dst_host_cnn_balance
+{
+	struct sockaddr_in addr;
+	char srvaddr[64];
+	
+	struct event timer_detect;
+	struct bufferevent *bev;
+
+	u_int32_t dstip;
+	enum TCP_CONNECTION_STATUS connection_status;
+	bool connect_failed;
+
+	struct doris_http_parameter *param;
+};
+
+struct doris_http_parameter
+{
+	u_int32_t server_port;
+	u_int32_t manage_port;
+	int32_t timer_period;
+	u_int32_t max_http_sessions;
+	int32_t connected_hosts;
+	int32_t failed_hosts;
+	int32_t ssl_connection;
+
+	struct dst_ipaddr_group ipgroup;
+	struct consistent_hash *conhash;
+	struct dst_host_cnn_balance *balance;
+
+	long maximum_host_cnns;
+	long transfer_timeout;//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+	long maximum_pipelines;
+	void *runtime_log;
+	struct event_base* evbase;
+};
+
+struct doris_http_instance
+{
+	struct event_base* evbase;
+	SSL_CTX *ssl_instance;
+	void *privdata;
+
+	map<u_int32_t, doris_curl_multihd*> *server_hosts;
+
+	struct doris_http_parameter *param;
+	void *runtime_log;
+};
+
+struct time_event
+{
+	struct event timer_event;
+	void *data;
+};
+
+int32_t param_get_connected_hosts(struct doris_http_parameter *param);
+int32_t param_get_failed_hosts(struct doris_http_parameter *param);
+
+struct doris_http_parameter *doris_http_parameter_new(const char* profile_path, const char* section, struct event_base* evbase, void *runtime_log);
+struct doris_http_instance *doris_http_instance_new(struct doris_http_parameter *param, struct event_base* evbase, void *runtimelog);
+
+#endif
+

client/doris_client_transfer.cpp

@@ -0,0 +1,379 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h> 
+#include <stdio.h> 
+#include <stdlib.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/time.h>
+#include <time.h>
+#include <string.h>
+#include <openssl/md5.h>
+
+#include "doris_client_http.h"
+#include "doris_client_transfer.h"
+#include "nirvana_conhash.h"
+
+void doris_http_ctx_reset(struct doris_http_ctx *ctx, struct doris_http_callback *cb)
+{
+	struct doris_curl_multihd *multidata=ctx->multidata;
+	struct doris_http_instance *instance=ctx->instance;
+
+	if(ctx->curl != NULL)
+	{
+		curl_multi_remove_handle(ctx->multidata->multi_hd, ctx->curl);
+		curl_easy_cleanup(ctx->curl);
+		ctx->curl = NULL;
+	}
+	if(ctx->headers != NULL)
+	{
+		curl_slist_free_all(ctx->headers);
+	}
+	memset(ctx, 0, sizeof(struct doris_http_ctx));
+	ctx->multidata = multidata;
+	ctx->instance = instance;
+	ctx->cb = *cb;
+}
+
+void doris_http_ctx_destroy(struct doris_http_ctx *ctx)
+{
+	if(ctx->curl != NULL)
+	{
+		curl_multi_remove_handle(ctx->multidata->multi_hd, ctx->curl);
+		curl_easy_cleanup(ctx->curl);
+	}
+	if(ctx->headers != NULL)
+	{
+		curl_slist_free_all(ctx->headers);
+	}
+	free(ctx);
+}
+
+struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, struct doris_http_callback *cb, u_int64_t balance_seed)
+{
+	struct doris_http_ctx *ctx;
+	struct doris_curl_multihd *multidata;
+	struct conhash_bucket result;
+
+	if(CONHASH_OK != conhash_lookup_bucket_int(instance->param->conhash, balance_seed, &result))
+	{
+		return NULL;
+	}
+	assert(instance->server_hosts->find(result.bucket_id) != instance->server_hosts->end());
+	multidata = instance->server_hosts->find(result.bucket_id)->second;
+
+	ctx = (struct doris_http_ctx *)calloc(1, sizeof(struct doris_http_ctx));
+	ctx->instance = instance;
+	ctx->multidata = multidata;
+	ctx->cb = *cb;
+	return ctx;
+}
+
+long long caculate_http_sessions_sum(const struct doris_http_instance *instance)
+{
+	map<u_int32_t, doris_curl_multihd*>::iterator iter;
+	long long sessions = 0;
+
+	if(instance == NULL) return 0;
+	for(iter=instance->server_hosts->begin(); iter!=instance->server_hosts->end(); iter++)
+	{
+		sessions += iter->second->sessions;
+	}
+	return sessions;
+}
+
+
+static inline void curl_set_common_options(CURL *curl, long transfer_timeout, char *errorbuf)
+{
+	curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf);
+	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
+	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 1000L); 
+	curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //<2F><><EFBFBD>Է<EFBFBD><D4B7>ֶ<EFBFBD><D6B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ij<EFBFBD><C4B3><EFBFBD>ӽ<EFBFBD><D3BD>տ<EFBFBD>ס<EFBFBD><D7A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+	//ctx->error="Operation too slow. Less than 100 bytes/sec transferred the last 10 seconds"
+	curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 10L);
+	curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
+	curl_easy_setopt(curl, CURLOPT_USERAGENT, "Doris Client Linux X64");
+}
+
+static size_t curl_response_write_cb(void *ptr, size_t size, size_t count, void *userp)
+{
+	struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp;
+	CURLcode code=CURLE_OK;
+
+	if(ctx->res_code == 0) //<2F>״<EFBFBD>Ӧ<EFBFBD><D3A6>ʱ<EFBFBD>ȿ<EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>200
+	{
+		code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code);
+	}
+	if(ctx->cb.write_cb != NULL)
+	{
+		ctx->cb.write_cb((const char*)ptr, size*count, code, ctx->res_code, ctx->cb.userp);
+	}
+	return size*count;
+}
+
+static size_t curl_response_header_cb(void *ptr, size_t size, size_t count, void *userp)
+{
+	struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp;
+	size_t len=size*count;
+	CURLcode code=CURLE_OK;
+	
+	if(ctx->res_code == 0) //<2F>״<EFBFBD>Ӧ<EFBFBD><D3A6>ʱ<EFBFBD>ȿ<EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>200
+	{
+		code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code);
+	}
+	if(ctx->cb.header_cb != NULL)
+	{
+		ctx->cb.header_cb((const char*)ptr, len, code, ctx->res_code, ctx->cb.userp);
+	}
+	return len;
+}
+
+void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header)
+{
+	ctx->headers = curl_slist_append(ctx->headers, header);
+}
+
+int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri)
+{	
+	char minio_url[2048];
+
+	assert(ctx->curl == NULL);
+	if(NULL == (ctx->curl=curl_easy_init())) 
+	{
+		return -1;
+	}
+
+	if(ctx->instance->param->ssl_connection)
+	{
+		snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri);
+		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L);
+		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L);
+	}
+	else
+	{
+		snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri);
+	}
+	curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
+	
+	if(ctx->headers != NULL) //LOSFģʽ<C4A3><CABD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ģʽ<C4A3>°<EFBFBD><C2B0><EFBFBD>Range<67><65>ȡ
+	{
+		curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
+	}
+	curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb);
+	curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
+	curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb);
+	curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
+	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
+	curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
+
+	if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
+	{
+		assert(0);
+		return -2;
+	}
+	ctx->transfering = 1;
+	return 0;
+}
+
+int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, const char *data, size_t data_len)
+{
+	char minio_url[2048];
+
+	assert(ctx->curl == NULL);
+	if(NULL == (ctx->curl=curl_easy_init())) 
+	{
+		return -1;
+	}
+
+	doris_http_ctx_add_header(ctx, "Expect:"); //ע<><D7A2>POST<53><54><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Expect<63><74>ϵ<EFBFBD><CFB5>Ҫ<EFBFBD><D2AA>ȷ<EFBFBD><C8B7><EFBFBD><EFBFBD>CURLOPT_POSTFIELDSIZE
+	if(ctx->instance->param->ssl_connection)
+	{
+		snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri);
+		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L);
+		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L);
+	}
+	else
+	{
+		snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri);
+	}
+	curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
+	curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
+	curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, data_len); //<2F><><EFBFBD><EFBFBD>Content-Length<74><68><EFBFBD><EFBFBD>CURLOPT_COPYPOSTFIELDS֮ǰ<D6AE><C7B0><EFBFBD><EFBFBD>
+	if(data_len > 0)
+	{
+		curl_easy_setopt(ctx->curl, CURLOPT_COPYPOSTFIELDS, data);
+	}
+	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
+	curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb);
+	curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
+	if(ctx->cb.header_cb != NULL)
+	{
+		curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb);
+		curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
+	}
+	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
+	curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);
+
+	if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
+	{
+		assert(0);
+		return -2;
+	}
+	ctx->transfering = 1;
+	return 0;
+}
+
+static void check_multi_info(CURLM *multi)
+{
+	CURLMsg *msg;
+	int msgs_left;
+	struct doris_http_ctx *ctx;
+	CURL *easy;
+	CURLcode res;
+
+	while((msg = curl_multi_info_read(multi, &msgs_left))) 
+	{
+		if(msg->msg != CURLMSG_DONE)
+		{
+			continue;
+		}
+		
+		easy = msg->easy_handle;
+		res = msg->data.result;
+		curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ctx);
+		curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &ctx->res_code);
+		curl_multi_remove_handle(multi, easy);
+		curl_easy_cleanup(easy);
+		ctx->curl = NULL;
+		ctx->transfering = 0;
+		ctx->res = res;
+
+		ctx->cb.transfer_done_cb(ctx->res, ctx->res_code, ctx->error, ctx->cb.userp);
+	}
+}
+
+/* Called by libevent when we get action on a multi socket */
+static void libevent_socket_event_cb(int fd, short action, void *userp)
+{
+	struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; //from event_assign
+	CURLMcode rc;
+	int what, still_running;
+
+	what = ((action&EV_READ)?CURL_CSELECT_IN:0) | ((action & EV_WRITE)?CURL_CSELECT_OUT:0);
+
+	rc = curl_multi_socket_action(multidata->multi_hd, fd, what, &still_running);
+	multidata->sessions = still_running;
+	assert(rc==CURLM_OK);
+
+	check_multi_info(multidata->multi_hd);
+	if(still_running<=0 && evtimer_pending(&multidata->timer_event, NULL))
+	{
+		evtimer_del(&multidata->timer_event);
+	}
+}
+
+/* Called by libevent when our timeout expires */
+static void libevent_timer_event_cb(int fd, short kind, void *userp)
+{
+	struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp;
+	CURLMcode rc;
+	int still_running;
+
+	rc = curl_multi_socket_action(multidata->multi_hd, CURL_SOCKET_TIMEOUT, 0, &still_running);	
+	multidata->sessions = still_running;
+	assert(rc==CURLM_OK);
+	check_multi_info(multidata->multi_hd);
+}
+
+static int curl_socket_function_cb(CURL *curl, curl_socket_t sockfd, int what, void *userp, void *sockp)
+{
+	struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp;
+	struct curl_socket_data *sockinfo = (struct curl_socket_data *)sockp; //curl_multi_assign, for socket
+	int action;
+
+	if(what == CURL_POLL_REMOVE)
+	{
+		if(sockinfo != NULL)
+		{
+			event_del(&sockinfo->sock_event);
+			free(sockinfo);
+		}
+	}
+	else
+	{
+		if(sockinfo == NULL)
+		{
+			sockinfo = (struct curl_socket_data *)calloc(1, sizeof(struct curl_socket_data));
+			curl_multi_assign(multidata->multi_hd, sockfd, sockinfo);
+		}
+		else
+		{
+			event_del(&sockinfo->sock_event);
+		}
+		
+		action = (what&CURL_POLL_IN?EV_READ:0)|(what&CURL_POLL_OUT?EV_WRITE:0)|EV_PERSIST;
+		event_assign(&sockinfo->sock_event, multidata->evbase, sockfd, action, libevent_socket_event_cb, multidata);
+		event_add(&sockinfo->sock_event, NULL);
+	}
+
+	return 0;
+}
+
+static int curl_timer_function_cb(CURLM *multi, long timeout_ms, void *userp)
+{
+	struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp;
+	struct timeval timeout;
+	CURLMcode rc;
+	int still_running;
+
+	timeout.tv_sec = timeout_ms/1000;
+	timeout.tv_usec = (timeout_ms%1000)*1000;
+
+	if(timeout_ms == 0)
+	{
+		//timeout_ms is 0 means we should call curl_multi_socket_action/curl_multi_perform at once.
+		//To initiate the whole process(inform CURLMOPT_SOCKETFUNCTION callback) or when timeout occurs.
+		rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running);
+		multidata->sessions = still_running;
+		assert(rc==CURLM_OK);
+	}
+	else if(timeout_ms == -1) //timeout_ms is -1 means we should delete the timer.
+	{
+		//call curl_multi_socket_action to update multidata->sessions, otherwise it will not be updated to 0
+		//when all transfers complete in some occasions(eg, GET, some objects hited while others miss).
+		rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running);
+		multidata->sessions = still_running;
+		evtimer_del(&multidata->timer_event);
+	}
+	else //update the timer to the new value.
+	{
+		evtimer_add(&multidata->timer_event, &timeout);
+	}
+	
+	return 0; //0-success; -1-error
+}
+
+
+//Ϊÿ<CEAA><C3BF>Minio Host<73><74><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ե<EFBFBD>curl multi handle
+struct doris_curl_multihd *doris_initialize_multihd_for_host(struct doris_http_instance *instance, struct dst_host_cnn_balance *host)
+{
+	struct doris_curl_multihd *multidata;
+
+	multidata = (struct doris_curl_multihd *)calloc(1, sizeof(struct doris_curl_multihd));
+	multidata->multi_hd = curl_multi_init();
+	multidata->evbase = instance->evbase;
+	multidata->host   = host;
+
+	curl_multi_setopt(multidata->multi_hd, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX);
+	curl_multi_setopt(multidata->multi_hd, CURLMOPT_MAX_HOST_CONNECTIONS, instance->param->maximum_host_cnns); 
+	curl_multi_setopt(multidata->multi_hd, CURLMOPT_MAX_PIPELINE_LENGTH, instance->param->maximum_pipelines); 
+	curl_multi_setopt(multidata->multi_hd, CURLMOPT_SOCKETFUNCTION, curl_socket_function_cb);
+	curl_multi_setopt(multidata->multi_hd, CURLMOPT_SOCKETDATA, multidata); //curl_socket_function_cb *userp
+	curl_multi_setopt(multidata->multi_hd, CURLMOPT_TIMERFUNCTION, curl_timer_function_cb);
+	curl_multi_setopt(multidata->multi_hd, CURLMOPT_TIMERDATA, multidata);
+	evtimer_assign(&multidata->timer_event, instance->evbase, libevent_timer_event_cb, multidata);
+	return multidata;
+}
+
+

client/doris_client_transfer.h

@@ -0,0 +1,64 @@
+#ifndef __DORIS_HTTP_TRANSFER_H__
+#define __DORIS_HTTP_TRANSFER_H__
+
+#include <event2/event.h>  
+#include <event.h>
+
+#include <curl/curl.h>
+
+struct doris_http_callback
+{
+	void *userp;
+	void (*header_cb)(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp); //HEAD/GET<45><54><EFBFBD>ɿ<EFBFBD>
+	void (*write_cb)(const char *ptr, size_t bytes, CURLcode code, long res_code, void *userp);  //GETʱ<54><CAB1>Ϊ<EFBFBD>գ<EFBFBD><D5A3><EFBFBD><EFBFBD><EFBFBD>Ϊ<EFBFBD><CEAA>ʱ<EFBFBD><CAB1><EFBFBD>ص<EFBFBD>
+	void (*read_process_cb)(void *data, size_t data_offset, void *userp); //PUTʱ<54><CAB1>Ϊ<EFBFBD><CEAA>ʱ<EFBFBD><CAB1><EFBFBD>ص<EFBFBD>
+	void (*transfer_done_cb)(CURLcode res, long res_code, const char *err, void *userp); //<2F><><EFBFBD>ɿ<EFBFBD>
+};
+
+struct doris_http_ctx
+{
+	struct doris_curl_multihd *multidata;
+	CURL *curl;
+	struct curl_slist *headers;
+	char error[CURL_ERROR_SIZE];	
+	struct doris_http_callback cb;
+	CURLcode res;
+	long res_code;
+	int32_t transfering;
+
+	char *put_data;
+	struct evbuffer *put_evbuf;
+	size_t put_length;
+	size_t put_offset;
+	struct doris_http_instance *instance;
+};
+
+struct dst_host_cnn_balance;
+struct doris_curl_multihd
+{
+	struct dst_host_cnn_balance *host;
+	volatile long long sessions;     //<2F><>ǰ<EFBFBD><C7B0><EFBFBD>ڽ<EFBFBD><DABD><EFBFBD>GET/PUT<55>ĻỰ<C4BB><E1BBB0>
+	CURLM *multi_hd;
+	struct event timer_event;        //curl<72><6C><EFBFBD><EFBFBD><EFBFBD>Ķ<EFBFBD>ʱ<EFBFBD><CAB1>
+	struct event_base* evbase;
+};
+
+struct doris_curl_multihd *doris_initialize_multihd_for_host(struct doris_http_instance *instance, struct dst_host_cnn_balance *host);
+struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, struct doris_http_callback *cb, u_int64_t balance_seed);
+void doris_http_ctx_destroy(struct doris_http_ctx *ctx);
+void doris_http_ctx_reset(struct doris_http_ctx *ctx, struct doris_http_callback *cb);
+
+void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header);
+
+int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri);
+int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, const char *data, size_t data_len);
+long long caculate_http_sessions_sum(const struct doris_http_instance *instance);
+
+
+struct curl_socket_data
+{
+	struct event sock_event;
+};
+
+#endif
+

client/nirvana_conhash.cpp

@@ -0,0 +1,482 @@
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <math.h>
+
+#include <map>
+
+#include "nirvana_murmurhash.h"
+#include "nirvana_conhash.h"
+
+using namespace std;
+
+#ifndef offsetof
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif
+
+struct ch_point
+{
+	u_int32_t bucket_id;
+	u_int32_t bucket_index; /* which backend it belongs to, use IP address */
+	u_int64_t hit_cnt;
+	u_int64_t point_val; /* hash code of this nodes, it is used to map node to consistent hash cycle */
+};
+
+struct ch_bucket_inner
+{
+	struct conhash_bucket bucket;
+
+	struct ch_point point_array[CONHASH_MAX_POINTS_PER_BUCKET]; //<2F><>ʹ<EFBFBD><CAB9>point_val<61><6C>Աȥ<D4B1><C8A5>
+	int32_t is_valid;
+	int32_t bucket_index;
+	u_int64_t hit_cnt;
+};
+
+struct consistent_hash
+{
+	struct ch_bucket_inner *bucket_array;
+	u_int32_t bucket_array_size;
+	u_int32_t bucket_cnt;
+	u_int32_t point_num;
+	struct ch_point *point_array;
+	map<u_int32_t, u_int32_t> *map_id_index;
+};
+
+//coefficient of variation of the RMSD
+double conhash_calulate_CVRSMD(struct consistent_hash *ch)
+{
+	struct ch_bucket_inner* b=NULL;
+	u_int32_t i=0;
+	double sum_hit=0, sum_point=0;
+	double MSE=0,RMSD=0,CVRMSD=0;
+	for(i=0;i<ch->bucket_array_size;i++)
+	{
+		b=ch->bucket_array+i;
+		if(b->is_valid==0)
+		{
+			continue;
+		}
+		sum_hit+=(double)b->hit_cnt;
+		sum_point+=(double)b->bucket.point_num;
+	}
+	for(i=0;i<ch->bucket_array_size;i++)
+	{
+		b=ch->bucket_array+i;
+		if(b->is_valid==0)
+		{
+			continue;
+		}
+		MSE+=pow(b->hit_cnt-(b->bucket.point_num*sum_hit)/sum_point,2);
+	}
+	RMSD = sqrt(MSE/ch->bucket_cnt);
+	CVRMSD = RMSD/(sum_hit/ch->bucket_cnt);
+	return CVRMSD;
+}
+
+static int qsort_cmp_by_key_increase(const void* a, const void* b)
+{
+	if(((const struct ch_point*)a)->point_val > ((const struct ch_point*)b)->point_val)
+	{
+		return 1;
+	}
+	else if(((const struct ch_point*)a)->point_val == ((const struct ch_point*)b)->point_val)
+	{
+		return 0;
+	}
+	else
+	{
+		return -1;
+	}
+}
+
+static int qsort_cmp_by_key_decrease(const void* a, const void* b)
+{
+	if(((const struct ch_point*)a)->point_val > ((const struct ch_point*)b)->point_val)
+	{
+		return -1;
+	}
+	else if(((const struct ch_point*)a)->point_val == ((const struct ch_point*)b)->point_val)
+	{
+		return 0;
+	}
+	else
+	{
+		return 1;
+	}
+}
+
+//	(vector<int>& nums, int target)
+static u_int32_t search_up_bound(u_int64_t target, const void *base,
+                     int32_t nmemb, size_t size,int val_offset)
+{
+    int32_t low = 0, high = nmemb-1, mid;
+
+    // Invariant: the desired index is between [low, high+1]
+    
+	while (low <= high)
+	{
+		mid = low + (high-low)/2;
+		if(*(u_int64_t*)((char*)base+size*mid+val_offset) < target)
+		{
+			low = mid+1;
+		}
+		else
+		{
+			high = mid-1;
+		}
+	}
+	if(low == nmemb)
+	{
+		low=0;
+	}
+    // (1) At this point, low > high. That is, low >= high+1
+    // (2) From the invariant, we know that the index is between [low, high+1], so low <= high+1. Follwing from (1), now we know low == high+1.
+    // (3) Following from (2), the index is between [low, high+1] = [low, low], which means that low is the desired index
+    //     Therefore, we return low as the answer. You can also return high+1 as the result, since low == high+1
+    return low;
+}
+
+//<2F><>֤<EFBFBD><D6A4>ͬ<EFBFBD><CDAC>bucket_id&&point_index<65><78><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC>point_id
+static u_int64_t bucket_gen_uniq_point(struct ch_bucket_inner *inner_bucket, u_int32_t cur_point_index)
+{
+	u_int64_t x=0, seed;
+	u_int32_t hash, i=0;
+
+	seed = (((u_int64_t)cur_point_index)<<32) | inner_bucket->bucket.bucket_id;
+	hash = murmurhash2(&seed, sizeof(u_int64_t), 23068673);
+	x = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id;
+
+	while(i != cur_point_index)
+	{
+		for(i=0; i<cur_point_index; i++)
+		{
+			if(x == inner_bucket->point_array[i].point_val) //<2F><>ͻ
+			{
+				seed = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id;
+				hash = murmurhash2(&seed, sizeof(u_int64_t), 23068673);
+				x = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id;
+				i = 0;
+				break;
+			}
+		}
+	}
+	inner_bucket->point_array[cur_point_index].point_val = x;
+	return x;
+}
+
+u_int32_t conhash_get_bucket_num(struct consistent_hash *ch)
+{
+	return ch->bucket_cnt;
+}
+
+struct consistent_hash *conhash_instance_new(const struct conhash_bucket *buckets, uint32_t bucket_num)
+{
+	struct consistent_hash *ch=NULL;
+	u_int32_t i, j, k;
+	u_int64_t randval;
+
+	for(i=0; i<bucket_num; i++)
+	{
+		if(buckets[i].point_num > CONHASH_MAX_POINTS_PER_BUCKET)
+		{
+			return NULL;
+		}
+	}
+	ch = (struct consistent_hash *)calloc(1, sizeof(struct consistent_hash));
+
+	/*buckets*/
+	ch->map_id_index = new map<u_int32_t, u_int32_t>;
+	ch->bucket_array = (struct ch_bucket_inner*)calloc(1, sizeof(struct ch_bucket_inner)*bucket_num);
+	for(i=0; i<bucket_num; i++)
+	{
+		memcpy(&(ch->bucket_array[i].bucket), &buckets[i], sizeof(struct conhash_bucket));
+		ch->bucket_array[i].is_valid = 1;
+		ch->bucket_array[i].bucket_index = i;
+		ch->point_num += buckets[i].point_num;
+		ch->map_id_index->insert(make_pair(buckets[i].bucket_id, i));
+	}
+	ch->bucket_cnt = bucket_num;
+
+	/*global points*/
+	ch->point_array = (struct ch_point*)calloc(1, sizeof(struct ch_point)*ch->point_num);
+	ch->bucket_array_size = bucket_num;
+	for(i=0, k=0; i<ch->bucket_array_size; i++)
+	{
+		for(j=0; j<ch->bucket_array[i].bucket.point_num; j++,k++)
+		{
+			randval = bucket_gen_uniq_point(&ch->bucket_array[i], j);
+			ch->point_array[k].bucket_id    = ch->bucket_array[i].bucket.bucket_id;
+			ch->point_array[k].bucket_index = i;
+			ch->point_array[k].point_val    = randval;
+			ch->point_array[k].hit_cnt      = 0;
+		}
+		qsort(ch->bucket_array[i].point_array, ch->bucket_array[i].bucket.point_num, sizeof(struct ch_point), qsort_cmp_by_key_decrease);
+	}
+    qsort(ch->point_array, ch->point_num, sizeof(struct ch_point), qsort_cmp_by_key_increase);
+	return ch;
+}
+
+void conhash_instance_free(struct consistent_hash *ch)
+{
+	free(ch->point_array);
+	free(ch->bucket_array);
+	delete ch->map_id_index;
+	free(ch);
+}
+
+struct consistent_hash *conhash_instance_copy(struct consistent_hash *ch)
+{
+	struct consistent_hash *copy;
+	
+	copy = (struct consistent_hash *)calloc(1, sizeof(struct consistent_hash));
+	copy->bucket_array_size = ch->bucket_array_size;
+	copy->bucket_cnt = ch->bucket_cnt;
+	copy->point_num  = ch->point_num;
+	copy->bucket_array = (struct ch_bucket_inner*)calloc(sizeof(struct ch_bucket_inner), ch->bucket_array_size);	
+	memcpy(copy->bucket_array, ch->bucket_array, sizeof(struct ch_bucket_inner)*ch->bucket_array_size);
+
+	copy->point_array = (struct ch_point*)calloc(sizeof(struct ch_point), copy->point_num);
+	memcpy(copy->point_array, ch->point_array, sizeof(struct ch_point)*copy->point_num);
+
+	copy->map_id_index = new map<u_int32_t, u_int32_t>;
+	copy->map_id_index->insert(ch->map_id_index->begin(), ch->map_id_index->end());
+	return copy;
+}
+
+static enum CONHASH_ERRCODE conhash_add_points(struct consistent_hash *ch, struct ch_bucket_inner *inner_bucket, int32_t add_points)
+{
+	u_int64_t randval;
+
+	if(inner_bucket->bucket.point_num + add_points > CONHASH_MAX_POINTS_PER_BUCKET)
+	{
+		assert(0);return CONHASH_ERR_INVALID_ARGS;
+	}
+
+	ch->point_array = (struct ch_point *)realloc(ch->point_array,sizeof(struct ch_point)*(ch->point_num+add_points));
+
+	for(u_int32_t i=ch->point_num; i<ch->point_num+add_points; i++)
+	{
+		randval = bucket_gen_uniq_point(inner_bucket, inner_bucket->bucket.point_num);
+		inner_bucket->bucket.point_num++;
+		ch->point_array[i].bucket_id = inner_bucket->bucket.bucket_id;
+		ch->point_array[i].bucket_index = inner_bucket->bucket_index;
+		ch->point_array[i].point_val = randval;
+		ch->point_array[i].hit_cnt = 0;
+	}
+	ch->point_num += add_points;
+	qsort(inner_bucket->point_array, inner_bucket->bucket.point_num, sizeof(struct ch_point), qsort_cmp_by_key_decrease);
+	qsort(ch->point_array, ch->point_num, sizeof(struct ch_point), qsort_cmp_by_key_increase);
+	return CONHASH_OK;
+}
+
+static enum CONHASH_ERRCODE conhash_del_points(struct consistent_hash *ch, struct ch_bucket_inner *inner_bucket, u_int32_t del_points)
+{
+	struct ch_point *tmp_points;
+	u_int32_t i, j, removed;
+
+	if(inner_bucket->bucket.point_num == 0)
+	{
+		return CONHASH_OK;
+	}
+	if(inner_bucket->bucket.point_num < del_points)
+	{
+		assert(0);return CONHASH_ERR_INVALID_ARGS;
+	}
+
+	tmp_points = (struct ch_point*)malloc(sizeof(struct ch_point)*ch->point_num);
+	memcpy(tmp_points, ch->point_array, sizeof(struct ch_point)*ch->point_num);
+
+	for(i=0,j=0,removed=0; i<ch->point_num; i++)
+	{
+		if(removed<del_points && tmp_points[i].bucket_id==inner_bucket->bucket.bucket_id)
+		{
+			assert(inner_bucket->point_array[inner_bucket->bucket.point_num-1].point_val == tmp_points[i].point_val);
+			inner_bucket->bucket.point_num--;
+			removed++;
+			continue;
+		}
+		memcpy(&ch->point_array[j], &tmp_points[i], sizeof(struct ch_point));
+		j++;
+	}
+	assert(removed == del_points);
+	free(tmp_points);
+	ch->point_num -= del_points;
+	//Sort is unnecessary after deletion.
+	return CONHASH_OK;
+}
+
+enum CONHASH_ERRCODE conhash_insert_bucket(struct consistent_hash *ch, const struct conhash_bucket *bucket)
+{
+	struct ch_bucket_inner *inner_bucket=NULL;
+	u_int32_t i, bucket_index;
+	map<u_int32_t, u_int32_t>::iterator iter;
+	enum CONHASH_ERRCODE code;
+	
+	if(bucket->point_num <= 0)
+	{
+		return CONHASH_ERR_INVALID_ARGS;
+	}
+	if((iter=ch->map_id_index->find(bucket->bucket_id)) != ch->map_id_index->end())
+	{
+		return CONHASH_BUCKET_ALREADY_EXIST;
+	}
+
+	if(ch->bucket_cnt < ch->bucket_array_size)
+	{
+		for(i=0; i<ch->bucket_array_size; i++)
+		{
+			if(ch->bucket_array[i].is_valid == 0)
+			{
+				bucket_index = i;
+				break;
+			}
+		}
+		assert(i < ch->bucket_array_size && ch->bucket_array[bucket_index].bucket.point_num==0); //һ<><D2BB><EFBFBD><EFBFBD><EFBFBD>ҵ<EFBFBD>
+	
+		inner_bucket = &ch->bucket_array[bucket_index];
+	}
+	else
+	{
+		assert(ch->bucket_array_size == ch->bucket_cnt);
+		bucket_index = ch->bucket_cnt;
+		ch->bucket_array_size = ch->bucket_cnt + 1;
+		ch->bucket_array = (struct ch_bucket_inner*)realloc(ch->bucket_array, sizeof(struct ch_bucket_inner)*ch->bucket_array_size);
+		memset(&ch->bucket_array[bucket_index], 0, sizeof(struct ch_bucket_inner));
+		inner_bucket = &ch->bucket_array[bucket_index];
+	}
+	inner_bucket->bucket.bucket_id = bucket->bucket_id;
+	inner_bucket->bucket.tag       = bucket->tag;
+
+	if(CONHASH_OK != (code=conhash_add_points(ch, inner_bucket, bucket->point_num)))
+	{
+		return code;
+	}
+	inner_bucket->is_valid = 1;
+	inner_bucket->bucket_index = bucket_index;
+	inner_bucket->hit_cnt = 0;
+	ch->bucket_cnt++;
+	ch->map_id_index->insert(make_pair(bucket->bucket_id, bucket_index));
+	return CONHASH_OK;
+}
+
+enum CONHASH_ERRCODE conhash_remove_bucket(struct consistent_hash *ch, u_int32_t bucket_id, void (*free_cb)(void *tag, u_int32_t point_num))
+{
+	struct ch_bucket_inner* inner_bucket=NULL;
+	u_int32_t bucket_index;
+	map<u_int32_t, u_int32_t>::iterator iter;
+	enum CONHASH_ERRCODE code;
+
+	if((iter=ch->map_id_index->find(bucket_id)) == ch->map_id_index->end())
+	{
+		return CONHASH_BUCKET_NOT_FOUND;
+	}
+	bucket_index = iter->second;
+	assert(bucket_index < ch->bucket_array_size);
+
+	inner_bucket = &ch->bucket_array[bucket_index];
+	if(CONHASH_OK != (code=conhash_del_points(ch, inner_bucket, inner_bucket->bucket.point_num)))
+	{
+		return code;
+	}
+	ch->bucket_cnt--;
+	inner_bucket->is_valid = 0;
+	if(free_cb)
+	{
+		free_cb(inner_bucket->bucket.tag, inner_bucket->bucket.point_num);
+	}
+	inner_bucket->bucket.point_num = 0;
+	inner_bucket->bucket.tag = NULL;
+	ch->map_id_index->erase(bucket_id);
+	return CONHASH_OK;
+}
+
+enum CONHASH_ERRCODE conhash_renew_bucket(struct consistent_hash *ch, struct conhash_bucket *bucket)
+{
+	struct ch_bucket_inner* inner_bucket=NULL;
+	u_int32_t bucket_index;
+	map<u_int32_t, u_int32_t>::iterator iter;
+
+	if((iter=ch->map_id_index->find(bucket->bucket_id)) == ch->map_id_index->end())
+	{
+		assert(0);return CONHASH_BUCKET_NOT_FOUND;
+	}
+	bucket_index = iter->second;
+	assert(bucket_index < ch->bucket_array_size);
+
+	inner_bucket = &ch->bucket_array[bucket_index];
+	assert(inner_bucket->is_valid == 1);
+	inner_bucket->bucket.tag = bucket->tag;
+
+	if(inner_bucket->bucket.point_num == bucket->point_num)
+	{
+		return CONHASH_OK;
+	}
+	else if(inner_bucket->bucket.point_num < bucket->point_num)
+	{
+		return conhash_add_points(ch, inner_bucket, bucket->point_num-inner_bucket->bucket.point_num);
+	}
+	else
+	{
+		return conhash_del_points(ch, inner_bucket, inner_bucket->bucket.point_num-bucket->point_num);
+	}
+}
+
+enum CONHASH_ERRCODE conhash_lookup_bucket(struct consistent_hash *ch, const void* key, int len, struct conhash_bucket* result)
+{
+	int idx=0, bucket_index=0;
+	u_int64_t hash;
+
+	if(ch->bucket_cnt == 0)
+	{
+		return CONHASH_NO_VALID_BUCKETS;
+	}
+
+	hash = MurmurHash64A(key, len, 23068673);
+	idx = search_up_bound(hash, ch->point_array, ch->point_num, sizeof(struct ch_point), offsetof(struct ch_point, point_val));
+	ch->point_array[idx].hit_cnt++;
+	bucket_index = ch->point_array[idx].bucket_index;
+	assert(ch->bucket_array[bucket_index].is_valid == 1);
+	ch->bucket_array[bucket_index].hit_cnt++;
+	memcpy(result, &(ch->bucket_array[bucket_index].bucket), sizeof(struct conhash_bucket));
+	return CONHASH_OK;
+}
+
+enum CONHASH_ERRCODE conhash_lookup_bucket_int(struct consistent_hash *ch, u_int64_t randint, struct conhash_bucket* result)
+{
+	int idx=0, bucket_index=0;
+
+	if(ch->bucket_cnt == 0)
+	{
+		return CONHASH_NO_VALID_BUCKETS;
+	}
+
+	idx = search_up_bound(randint, ch->point_array, ch->point_num, sizeof(struct ch_point), offsetof(struct ch_point, point_val));
+	ch->point_array[idx].hit_cnt++;
+	bucket_index = ch->point_array[idx].bucket_index;
+	assert(ch->bucket_array[bucket_index].is_valid == 1);
+	ch->bucket_array[bucket_index].hit_cnt++;
+	memcpy(result, &(ch->bucket_array[bucket_index].bucket), sizeof(struct conhash_bucket));
+	return CONHASH_OK;
+}
+
+void conhash_dump_detail(struct consistent_hash *ch)
+{
+	/*for(u_int32_t i=0; i<ch->point_num; i++)
+	{
+		printf("bucket_id: %10u, bucket_index: %5u, point_val:%lx, hit_cnt: %lu\n", ch->point_array[i].bucket_id,
+			ch->point_array[i].bucket_index, ch->point_array[i].point_val, ch->point_array[i].hit_cnt);
+	}
+
+	printf("\n\n\n\n");*/
+	for(u_int32_t i=0; i<ch->bucket_cnt; i++)
+	{
+		if(ch->bucket_array[i].is_valid)
+		{
+			printf("bucket_id: %10u, bucket_index: %5u, hit_cnt: %lu\n", ch->bucket_array[i].bucket.bucket_id,
+				i, ch->bucket_array[i].hit_cnt);
+		}
+	}
+}
+

client/nirvana_conhash.h

@@ -0,0 +1,50 @@
+#ifndef __NVN_CONSISTENT_HASH_H__
+#define __NVN_CONSISTENT_HASH_H__
+
+#include <stdint.h>
+
+#ifndef CONHASH_MAX_POINTS_PER_BUCKET
+#define CONHASH_MAX_POINTS_PER_BUCKET 128
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+enum CONHASH_ERRCODE
+{
+	CONHASH_OK = 0,
+	CONHASH_ERR_INVALID_ARGS = -1,
+	CONHASH_BUCKET_NOT_FOUND = -2,
+	CONHASH_BUCKET_ALREADY_EXIST=-3,
+	CONHASH_NO_VALID_BUCKETS=-4,
+};
+
+struct conhash_bucket
+{
+	uint32_t bucket_id;
+	uint32_t point_num;    /*should be not more than CONHASH_MAX_POINTS_PER_BUCKET*/
+	void* tag;
+};
+struct consistent_hash;
+
+/*API<50>̲߳<DFB3><CCB2><EFBFBD>ȫ*/
+struct consistent_hash *conhash_instance_new(const struct conhash_bucket *buckets, uint32_t bucket_num);
+void conhash_instance_free(struct consistent_hash *ch);
+struct consistent_hash *conhash_instance_copy(struct consistent_hash *ch);
+
+enum CONHASH_ERRCODE conhash_insert_bucket(struct consistent_hash *ch,const struct conhash_bucket* bucket);
+enum CONHASH_ERRCODE conhash_remove_bucket(struct consistent_hash *ch, u_int32_t bucket_id, void (*free_cb)(void *tag, u_int32_t point_num));
+enum CONHASH_ERRCODE conhash_renew_bucket(struct consistent_hash *ch, struct conhash_bucket* bucket); /*<2A><><EFBFBD><EFBFBD>point_num<75><6D>tag*/
+enum CONHASH_ERRCODE conhash_lookup_bucket(struct consistent_hash *ch, const void* key, int len, struct conhash_bucket *result/*OUT*/);
+enum CONHASH_ERRCODE conhash_lookup_bucket_int(struct consistent_hash *ch, u_int64_t randint, struct conhash_bucket* result);
+
+double conhash_calulate_CVRSMD(struct consistent_hash *p);
+u_int32_t conhash_get_bucket_num(struct consistent_hash *ch);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

client/nirvana_murmurhash.cpp

@@ -0,0 +1,99 @@
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "nirvana_murmurhash.h"
+
+unsigned int murmurhash2(const void * key, int len, const unsigned int seed)
+{
+	// 'm' and 'r' are mixing constants generated offline.
+	// They're not really 'magic', they just happen to work well.
+
+	const unsigned int m = 0x5bd1e995;
+	const int r = 24;
+
+	// Initialize the hash to a 'random' value
+
+	unsigned int h = seed ^ len;
+
+	// Mix 4 bytes at a time into the hash
+
+	const unsigned char * data = (const unsigned char *)key;
+
+	while(len >= 4)
+	{
+		unsigned int k = *(unsigned int *)data;
+
+		k *= m;
+		k ^= k >> r;
+		k *= m;
+
+		h *= m;
+		h ^= k;
+
+		data += 4;
+		len -= 4;
+	}
+
+	// Handle the last few bytes of the input array
+
+	switch(len)
+	{
+		case 3: h ^= data[2] << 16;
+		case 2: h ^= data[1] << 8;
+		case 1: h ^= data[0];
+		        h *= m;
+		default:break;
+	};
+
+	// Do a few final mixes of the hash to ensure the last few
+	// bytes are well-incorporated.
+
+	h ^= h >> 13;
+	h *= m;
+	h ^= h >> 15;
+	return h;
+}
+
+/*64-bit hash for 64-bit platforms*/
+u_int64_t MurmurHash64A(const void * key, int len, unsigned int seed)
+{
+	const uint64_t m = 0xc6a4a7935bd1e995;
+	const int r = 47;
+	uint64_t k, h = seed ^ (len * m);
+	const uint64_t * data = (const uint64_t *)key;
+	const uint64_t * end = data + (len/8);
+
+	while (data != end)
+	{
+		k = *data++;
+		k *= m;
+		k ^= k >> r;
+		k *= m;
+
+		h ^= k;
+		h *= m;
+	}
+
+	const unsigned char * data2 = (const unsigned char*)data;
+
+	switch (len & 7)
+	{
+		case 7: h ^= uint64_t(data2[6]) << 48;
+		case 6: h ^= uint64_t(data2[5]) << 40;
+		case 5: h ^= uint64_t(data2[4]) << 32;
+		case 4: h ^= uint64_t(data2[3]) << 24;
+		case 3: h ^= uint64_t(data2[2]) << 16;
+		case 2: h ^= uint64_t(data2[1]) << 8;
+		case 1: h ^= uint64_t(data2[0]);
+		h *= m;
+	};
+
+	h ^= h >> r;
+	h *= m;
+	h ^= h >> r;
+
+	return h;
+}
+
+

client/nirvana_murmurhash.h

@@ -0,0 +1,16 @@
+#ifndef __NVN_MURMURHASH_H__
+#define __NVN_MURMURHASH_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+unsigned int murmurhash2(const void * key, int len, const unsigned int seed);
+u_int64_t MurmurHash64A(const void * key, int len, unsigned int seed);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+