doris-doris-dispatch/client/doris_client_transfer.cpp

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h> 
#include <stdio.h> 
#include <stdlib.h>
#include <assert.h>
#include <errno.h>
#include <sys/time.h>
#include <time.h>
#include <string.h>
#include <openssl/md5.h>

#include "doris_client_http.h"
#include "doris_client_transfer.h"
#include "nirvana_conhash.h"

void doris_http_ctx_reset(struct doris_http_ctx *ctx, struct doris_http_callback *cb)
{
	struct doris_curl_multihd *multidata=ctx->multidata;
	struct doris_http_instance *instance=ctx->instance;

	if(ctx->curl != NULL)
	{
		curl_multi_remove_handle(ctx->multidata->multi_hd, ctx->curl);
		curl_easy_cleanup(ctx->curl);
		ctx->curl = NULL;
	}
	if(ctx->headers != NULL)
	{
		curl_slist_free_all(ctx->headers);
	}
	memset(ctx, 0, sizeof(struct doris_http_ctx));
	ctx->multidata = multidata;
	ctx->instance = instance;
	ctx->cb = *cb;
}

void doris_http_ctx_destroy(struct doris_http_ctx *ctx)
{
	if(ctx->curl != NULL)
	{
		curl_multi_remove_handle(ctx->multidata->multi_hd, ctx->curl);
		curl_easy_cleanup(ctx->curl);
	}
	if(ctx->headers != NULL)
	{
		curl_slist_free_all(ctx->headers);
	}
	free(ctx);
}

struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, 
	struct doris_http_callback *cb, u_int64_t balance_seed, char *host/*OUT*/, int32_t size)
{
	struct doris_http_ctx *ctx;
	struct doris_curl_multihd *multidata;
	struct conhash_bucket result;

	if(CONHASH_OK != conhash_lookup_bucket_int(instance->param->conhash, balance_seed, &result))
	{
		return NULL;
	}
	assert(instance->server_hosts->find(result.bucket_id) != instance->server_hosts->end());
	multidata = instance->server_hosts->find(result.bucket_id)->second;
	if(host != NULL)
	{
		snprintf(host, size, multidata->host->srvaddr);
	}
	ctx = (struct doris_http_ctx *)calloc(1, sizeof(struct doris_http_ctx));
	ctx->instance = instance;
	ctx->multidata = multidata;
	ctx->cb = *cb;
	return ctx;
}

long long caculate_http_sessions_sum(const struct doris_http_instance *instance)
{
	map<u_int32_t, doris_curl_multihd*>::iterator iter;
	long long sessions = 0;

	if(instance == NULL) return 0;
	for(iter=instance->server_hosts->begin(); iter!=instance->server_hosts->end(); iter++)
	{
		sessions += iter->second->sessions;
	}
	return sessions;
}


static inline void curl_set_common_options(CURL *curl, long transfer_timeout, char *errorbuf)
{
	curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf);
	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 1000L); 
	curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //<2F><><EFBFBD>Է<EFBFBD><D4B7>ֶ<EFBFBD><D6B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ij<EFBFBD><C4B3><EFBFBD>ӽ<EFBFBD><D3BD>տ<EFBFBD>ס<EFBFBD><D7A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
	//ctx->error="Operation too slow. Less than 100 bytes/sec transferred the last 30 seconds"
	curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L);
	curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L);
	curl_easy_setopt(curl, CURLOPT_USERAGENT, "Doris Client Linux X64");
}

static size_t curl_response_write_cb(void *ptr, size_t size, size_t count, void *userp)
{
	struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp;
	CURLcode code=CURLE_OK;

	if(ctx->res_code == 0) //<2F>״<EFBFBD>Ӧ<EFBFBD><D3A6>ʱ<EFBFBD>ȿ<EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>200
	{
		code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code);
	}
	if(ctx->cb.write_cb != NULL)
	{
		ctx->cb.write_cb((const char*)ptr, size*count, code, ctx->res_code, ctx->cb.userp);
	}
	return size*count;
}

static size_t curl_response_header_cb(void *ptr, size_t size, size_t count, void *userp)
{
	struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp;
	size_t len=size*count;
	CURLcode code=CURLE_OK;
	
	if(ctx->res_code == 0) //<2F>״<EFBFBD>Ӧ<EFBFBD><D3A6>ʱ<EFBFBD>ȿ<EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD>Ƿ<EFBFBD><C7B7><EFBFBD>200
	{
		code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code);
	}
	if(ctx->cb.header_cb != NULL)
	{
		ctx->cb.header_cb((const char*)ptr, len, code, ctx->res_code, ctx->cb.userp);
	}
	return len;
}

void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header)
{
	ctx->headers = curl_slist_append(ctx->headers, header);
}

/*maximum length 1024*/
void doris_http_ctx_add_header_kvstr(struct doris_http_ctx *ctx, const char *headername, const char *value)
{
	char header[1024];
	snprintf(header, 1024, "%s: %s", headername, value);
	ctx->headers = curl_slist_append(ctx->headers, header);
}

void doris_http_ctx_add_header_kvint(struct doris_http_ctx *ctx, const char *headername, u_int64_t value)
{
	char header[1024];
	snprintf(header, 1024, "%s: %lu", headername, value);
	ctx->headers = curl_slist_append(ctx->headers, header);
}

int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri)
{	
	char minio_url[2048];

	assert(ctx->curl == NULL);
	if(NULL == (ctx->curl=curl_easy_init())) 
	{
		assert(0);return -1;
	}

	if(ctx->instance->param->ssl_connection)
	{
		snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri);
		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L);
		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L);
	}
	else
	{
		snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
	
	if(ctx->headers != NULL) //LOSFģʽ<C4A3><CABD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ģʽ<C4A3>°<EFBFBD><C2B0><EFBFBD>Range<67><65>ȡ
	{
		curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb);
	curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
	curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb);
	curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
	curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);

	if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
	{
		assert(0);return -2;
	}
	ctx->transfering = 1;
	return 0;
}

int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, const char *data, size_t data_len)
{
	char minio_url[2048];

	assert(ctx->curl == NULL);
	if(NULL == (ctx->curl=curl_easy_init())) 
	{
		assert(0);return -1;
	}

	doris_http_ctx_add_header(ctx, "Expect:"); //ע<><D7A2>POST<53><54><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Expect<63><74>ϵ<EFBFBD><CFB5>Ҫ<EFBFBD><D2AA>ȷ<EFBFBD><C8B7><EFBFBD><EFBFBD>CURLOPT_POSTFIELDSIZE
	if(ctx->instance->param->ssl_connection)
	{
		snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri);
		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L);
		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L);
	}
	else
	{
		snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L);
	curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
	curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, data_len); //<2F><><EFBFBD><EFBFBD>Content-Length<74><68><EFBFBD><EFBFBD>CURLOPT_COPYPOSTFIELDS֮ǰ<D6AE><C7B0><EFBFBD><EFBFBD>
	if(data_len > 0)
	{
		curl_easy_setopt(ctx->curl, CURLOPT_COPYPOSTFIELDS, data);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
	curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb);
	curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
	if(ctx->cb.header_cb != NULL)
	{
		curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb);
		curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
	curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);

	if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
	{
		assert(0);return -2;
	}
	ctx->transfering = 1;
	return 0;
}

static size_t curl_put_data_request_send_cb(void *ptr, size_t size, size_t count, void *userp)
{
	size_t len;
	struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp;

	if(size==0 || count==0 || ctx->put_offset>=ctx->put_length)
	{
		return 0; //<2F><>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
	}

	len = ctx->put_length - ctx->put_offset; //ʣ<><CAA3><EFBFBD><EFBFBD><EFBFBD>ϴ<EFBFBD><CFB4>ij<EFBFBD><C4B3><EFBFBD>
	if(len > size * count)
	{
		len = size * count;
	}

	memcpy(ptr, ctx->put_data + ctx->put_offset, len);
	ctx->put_offset += len;

	if(ctx->cb.read_process_cb != NULL)
	{
		ctx->cb.read_process_cb(ctx->put_data, ctx->put_offset, ctx->cb.userp);
	}
	return len;
}

int doris_http_launch_put_request_data(struct doris_http_ctx *ctx, const char *uri, char *data, size_t data_len)
{
	char minio_url[2048];

	assert(ctx->curl == NULL);
	if(NULL == (ctx->curl=curl_easy_init())) 
	{
		assert(0);return -1;
	}
	ctx->put_data = data;
	ctx->put_length = data_len;
	ctx->put_offset = 0;

	if(ctx->instance->param->ssl_connection)
	{
		snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri);
		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L);
		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L);
	}
	else
	{
		snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
	curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
	curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb);
	curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
	if(ctx->cb.header_cb != NULL)
	{
		curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb);
		curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put_length);
	curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_data_request_send_cb);
	curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);

	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
	curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);

	if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
	{
		assert(0);return -2;
	}
	ctx->transfering = 1;
	return 0;
}

static size_t curl_put_evbuf_request_send_cb(void *ptr, size_t size, size_t count, void *userp)
{
	size_t len, space=size*count, send_len;
	struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp;

	if(size==0 || count==0 || ctx->put_offset>=ctx->put_length)
	{
		return 0;
	}

	len = ctx->put_length - ctx->put_offset;
	if(len > space)
	{
		len = space;
	}

	send_len = evbuffer_remove(ctx->put_evbuf, ptr, len);
	assert(send_len>0);
	ctx->put_offset += send_len;

	ctx->cb.read_process_cb(ctx->put_evbuf, ctx->put_offset, ctx->cb.userp);
	return send_len;
}

int doris_http_launch_put_request_evbuf(struct doris_http_ctx *ctx, const char *uri, struct evbuffer *evbuf, size_t data_len)
{
	char minio_url[2048];

	assert(ctx->curl == NULL);
	if(NULL == (ctx->curl=curl_easy_init())) 
	{
		assert(0);return -1;
	}
	ctx->put_evbuf = evbuf;
	ctx->put_length = data_len;
	ctx->put_offset = 0;

	if(ctx->instance->param->ssl_connection)
	{
		snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri);
		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L);
		curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L);
	}
	else
	{
		snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L);
	curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url);
	curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb);
	curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx);
	if(ctx->cb.header_cb != NULL)
	{
		curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb);
		curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx);
	}
	curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put_length);
	curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_evbuf_request_send_cb);
	curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx);

	curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx);
	curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers);
	curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error);

	if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl))
	{
		assert(0);return -2;
	}
	ctx->transfering = 1;
	return 0;
}

static void check_multi_info(CURLM *multi)
{
	CURLMsg *msg;
	int msgs_left;
	struct doris_http_ctx *ctx;
	CURL *easy;
	CURLcode res;

	while((msg = curl_multi_info_read(multi, &msgs_left))) 
	{
		if(msg->msg != CURLMSG_DONE)
		{
			continue;
		}
		
		easy = msg->easy_handle;
		res = msg->data.result;
		curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ctx);
		curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &ctx->res_code);
		curl_multi_remove_handle(multi, easy);
		curl_easy_cleanup(easy);
		ctx->curl = NULL;
		ctx->transfering = 0;
		ctx->res = res;

		ctx->cb.transfer_done_cb(ctx->res, ctx->res_code, ctx->error, ctx->cb.userp);
	}
}

/* Called by libevent when we get action on a multi socket */
static void libevent_socket_event_cb(int fd, short action, void *userp)
{
	struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; //from event_assign
	CURLMcode rc;
	int what, still_running;

	what = ((action&EV_READ)?CURL_CSELECT_IN:0) | ((action & EV_WRITE)?CURL_CSELECT_OUT:0);

	rc = curl_multi_socket_action(multidata->multi_hd, fd, what, &still_running);
	multidata->sessions = still_running;
	assert(rc==CURLM_OK);

	check_multi_info(multidata->multi_hd);
	if(still_running<=0 && evtimer_pending(&multidata->timer_event, NULL))
	{
		evtimer_del(&multidata->timer_event);
	}
}

/* Called by libevent when our timeout expires */
static void libevent_timer_event_cb(int fd, short kind, void *userp)
{
	struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp;
	CURLMcode rc;
	int still_running;

	rc = curl_multi_socket_action(multidata->multi_hd, CURL_SOCKET_TIMEOUT, 0, &still_running);	
	multidata->sessions = still_running;
	assert(rc==CURLM_OK);
	check_multi_info(multidata->multi_hd);
}

static int curl_socket_function_cb(CURL *curl, curl_socket_t sockfd, int what, void *userp, void *sockp)
{
	struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp;
	struct curl_socket_data *sockinfo = (struct curl_socket_data *)sockp; //curl_multi_assign, for socket
	int action;

	if(what == CURL_POLL_REMOVE)
	{
		if(sockinfo != NULL)
		{
			event_del(&sockinfo->sock_event);
			free(sockinfo);
		}
	}
	else
	{
		if(sockinfo == NULL)
		{
			sockinfo = (struct curl_socket_data *)calloc(1, sizeof(struct curl_socket_data));
			curl_multi_assign(multidata->multi_hd, sockfd, sockinfo);
		}
		else
		{
			event_del(&sockinfo->sock_event);
		}
		
		action = (what&CURL_POLL_IN?EV_READ:0)|(what&CURL_POLL_OUT?EV_WRITE:0)|EV_PERSIST;
		event_assign(&sockinfo->sock_event, multidata->evbase, sockfd, action, libevent_socket_event_cb, multidata);
		event_add(&sockinfo->sock_event, NULL);
	}

	return 0;
}

static int curl_timer_function_cb(CURLM *multi, long timeout_ms, void *userp)
{
	struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp;
	struct timeval timeout;
	CURLMcode rc;
	int still_running;

	timeout.tv_sec = timeout_ms/1000;
	timeout.tv_usec = (timeout_ms%1000)*1000;

	if(timeout_ms == 0)
	{
		//timeout_ms is 0 means we should call curl_multi_socket_action/curl_multi_perform at once.
		//To initiate the whole process(inform CURLMOPT_SOCKETFUNCTION callback) or when timeout occurs.
		rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running);
		multidata->sessions = still_running;
		assert(rc==CURLM_OK);
	}
	else if(timeout_ms == -1) //timeout_ms is -1 means we should delete the timer.
	{
		//call curl_multi_socket_action to update multidata->sessions, otherwise it will not be updated to 0
		//when all transfers complete in some occasions(eg, GET, some objects hited while others miss).
		rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running);
		multidata->sessions = still_running;
		evtimer_del(&multidata->timer_event);
	}
	else //update the timer to the new value.
	{
		evtimer_add(&multidata->timer_event, &timeout);
	}
	
	return 0; //0-success; -1-error
}


//Ϊÿ<CEAA><C3BF>Minio Host<73><74><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ե<EFBFBD>curl multi handle
struct doris_curl_multihd *doris_initialize_multihd_for_host(struct doris_http_instance *instance, struct dst_host_cnn_balance *host)
{
	struct doris_curl_multihd *multidata;

	multidata = (struct doris_curl_multihd *)calloc(1, sizeof(struct doris_curl_multihd));
	multidata->multi_hd = curl_multi_init();
	multidata->evbase = instance->evbase;
	multidata->host   = host;

	curl_multi_setopt(multidata->multi_hd, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX);
	curl_multi_setopt(multidata->multi_hd, CURLMOPT_MAX_HOST_CONNECTIONS, instance->param->maximum_host_cnns); 
	curl_multi_setopt(multidata->multi_hd, CURLMOPT_MAX_PIPELINE_LENGTH, instance->param->maximum_pipelines); 
	curl_multi_setopt(multidata->multi_hd, CURLMOPT_SOCKETFUNCTION, curl_socket_function_cb);
	curl_multi_setopt(multidata->multi_hd, CURLMOPT_SOCKETDATA, multidata); //curl_socket_function_cb *userp
	curl_multi_setopt(multidata->multi_hd, CURLMOPT_TIMERFUNCTION, curl_timer_function_cb);
	curl_multi_setopt(multidata->multi_hd, CURLMOPT_TIMERDATA, multidata);
	evtimer_assign(&multidata->timer_event, instance->evbase, libevent_timer_event_cb, multidata);
	return multidata;
}