#include #include #include #include #include #include #include #include #include #include #include #include "doris_client_http.h" #include "doris_client_transfer.h" #include "nirvana_conhash.h" void doris_http_ctx_reset(struct doris_http_ctx *ctx, struct doris_http_callback *cb) { struct doris_curl_multihd *multidata=ctx->multidata; struct doris_http_instance *instance=ctx->instance; if(ctx->curl != NULL) { curl_multi_remove_handle(ctx->multidata->multi_hd, ctx->curl); curl_easy_cleanup(ctx->curl); ctx->curl = NULL; } if(ctx->headers != NULL) { curl_slist_free_all(ctx->headers); } memset(ctx, 0, sizeof(struct doris_http_ctx)); ctx->multidata = multidata; ctx->instance = instance; ctx->cb = *cb; } void doris_http_ctx_destroy(struct doris_http_ctx *ctx) { if(ctx->curl != NULL) { curl_multi_remove_handle(ctx->multidata->multi_hd, ctx->curl); curl_easy_cleanup(ctx->curl); } if(ctx->headers != NULL) { curl_slist_free_all(ctx->headers); } free(ctx); } struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, struct doris_http_callback *cb, u_int64_t balance_seed) { struct doris_http_ctx *ctx; struct doris_curl_multihd *multidata; struct conhash_bucket result; if(CONHASH_OK != conhash_lookup_bucket_int(instance->param->conhash, balance_seed, &result)) { return NULL; } assert(instance->server_hosts->find(result.bucket_id) != instance->server_hosts->end()); multidata = instance->server_hosts->find(result.bucket_id)->second; ctx = (struct doris_http_ctx *)calloc(1, sizeof(struct doris_http_ctx)); ctx->instance = instance; ctx->multidata = multidata; ctx->cb = *cb; return ctx; } long long caculate_http_sessions_sum(const struct doris_http_instance *instance) { map::iterator iter; long long sessions = 0; if(instance == NULL) return 0; for(iter=instance->server_hosts->begin(); iter!=instance->server_hosts->end(); iter++) { sessions += iter->second->sessions; } return sessions; } static inline void curl_set_common_options(CURL *curl, long transfer_timeout, char *errorbuf) { curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 1000L); curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //测试发现多链接有某链接接收卡住的情况 //ctx->error="Operation too slow. Less than 100 bytes/sec transferred the last 10 seconds" curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 10L); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L); curl_easy_setopt(curl, CURLOPT_USERAGENT, "Doris Client Linux X64"); } static size_t curl_response_write_cb(void *ptr, size_t size, size_t count, void *userp) { struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp; CURLcode code=CURLE_OK; if(ctx->res_code == 0) //首次应答时先看应答码是否是200 { code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code); } if(ctx->cb.write_cb != NULL) { ctx->cb.write_cb((const char*)ptr, size*count, code, ctx->res_code, ctx->cb.userp); } return size*count; } static size_t curl_response_header_cb(void *ptr, size_t size, size_t count, void *userp) { struct doris_http_ctx *ctx = (struct doris_http_ctx *)userp; size_t len=size*count; CURLcode code=CURLE_OK; if(ctx->res_code == 0) //首次应答时先看应答码是否是200 { code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code); } if(ctx->cb.header_cb != NULL) { ctx->cb.header_cb((const char*)ptr, len, code, ctx->res_code, ctx->cb.userp); } return len; } void doris_http_ctx_add_header(struct doris_http_ctx *ctx, const char *header) { ctx->headers = curl_slist_append(ctx->headers, header); } int doris_http_launch_get_request(struct doris_http_ctx *ctx, const char *uri) { char minio_url[2048]; assert(ctx->curl == NULL); if(NULL == (ctx->curl=curl_easy_init())) { return -1; } if(ctx->instance->param->ssl_connection) { snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri); curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L); } else { snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri); } curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); if(ctx->headers != NULL) //LOSF模式或者正常模式下按照Range获取 { curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); } curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb); curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl)) { assert(0); return -2; } ctx->transfering = 1; return 0; } int doris_http_launch_post_request(struct doris_http_ctx *ctx, const char *uri, const char *data, size_t data_len) { char minio_url[2048]; assert(ctx->curl == NULL); if(NULL == (ctx->curl=curl_easy_init())) { return -1; } doris_http_ctx_add_header(ctx, "Expect:"); //注意POST方法与Expect关系,要明确给出CURLOPT_POSTFIELDSIZE if(ctx->instance->param->ssl_connection) { snprintf(minio_url, sizeof(minio_url), "https://%s/%s", ctx->multidata->host->srvaddr, uri); curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(ctx->curl, CURLOPT_SSL_VERIFYHOST, 0L); } else { snprintf(minio_url, sizeof(minio_url), "http://%s/%s", ctx->multidata->host->srvaddr, uri); } curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, data_len); //填充Content-Length,在CURLOPT_COPYPOSTFIELDS之前设置 if(data_len > 0) { curl_easy_setopt(ctx->curl, CURLOPT_COPYPOSTFIELDS, data); } curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_write_cb); curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); if(ctx->cb.header_cb != NULL) { curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_response_header_cb); curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); } curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); if(CURLM_OK != curl_multi_add_handle(ctx->multidata->multi_hd, ctx->curl)) { assert(0); return -2; } ctx->transfering = 1; return 0; } static void check_multi_info(CURLM *multi) { CURLMsg *msg; int msgs_left; struct doris_http_ctx *ctx; CURL *easy; CURLcode res; while((msg = curl_multi_info_read(multi, &msgs_left))) { if(msg->msg != CURLMSG_DONE) { continue; } easy = msg->easy_handle; res = msg->data.result; curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ctx); curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &ctx->res_code); curl_multi_remove_handle(multi, easy); curl_easy_cleanup(easy); ctx->curl = NULL; ctx->transfering = 0; ctx->res = res; ctx->cb.transfer_done_cb(ctx->res, ctx->res_code, ctx->error, ctx->cb.userp); } } /* Called by libevent when we get action on a multi socket */ static void libevent_socket_event_cb(int fd, short action, void *userp) { struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; //from event_assign CURLMcode rc; int what, still_running; what = ((action&EV_READ)?CURL_CSELECT_IN:0) | ((action & EV_WRITE)?CURL_CSELECT_OUT:0); rc = curl_multi_socket_action(multidata->multi_hd, fd, what, &still_running); multidata->sessions = still_running; assert(rc==CURLM_OK); check_multi_info(multidata->multi_hd); if(still_running<=0 && evtimer_pending(&multidata->timer_event, NULL)) { evtimer_del(&multidata->timer_event); } } /* Called by libevent when our timeout expires */ static void libevent_timer_event_cb(int fd, short kind, void *userp) { struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; CURLMcode rc; int still_running; rc = curl_multi_socket_action(multidata->multi_hd, CURL_SOCKET_TIMEOUT, 0, &still_running); multidata->sessions = still_running; assert(rc==CURLM_OK); check_multi_info(multidata->multi_hd); } static int curl_socket_function_cb(CURL *curl, curl_socket_t sockfd, int what, void *userp, void *sockp) { struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; struct curl_socket_data *sockinfo = (struct curl_socket_data *)sockp; //curl_multi_assign, for socket int action; if(what == CURL_POLL_REMOVE) { if(sockinfo != NULL) { event_del(&sockinfo->sock_event); free(sockinfo); } } else { if(sockinfo == NULL) { sockinfo = (struct curl_socket_data *)calloc(1, sizeof(struct curl_socket_data)); curl_multi_assign(multidata->multi_hd, sockfd, sockinfo); } else { event_del(&sockinfo->sock_event); } action = (what&CURL_POLL_IN?EV_READ:0)|(what&CURL_POLL_OUT?EV_WRITE:0)|EV_PERSIST; event_assign(&sockinfo->sock_event, multidata->evbase, sockfd, action, libevent_socket_event_cb, multidata); event_add(&sockinfo->sock_event, NULL); } return 0; } static int curl_timer_function_cb(CURLM *multi, long timeout_ms, void *userp) { struct doris_curl_multihd *multidata = (struct doris_curl_multihd *)userp; struct timeval timeout; CURLMcode rc; int still_running; timeout.tv_sec = timeout_ms/1000; timeout.tv_usec = (timeout_ms%1000)*1000; if(timeout_ms == 0) { //timeout_ms is 0 means we should call curl_multi_socket_action/curl_multi_perform at once. //To initiate the whole process(inform CURLMOPT_SOCKETFUNCTION callback) or when timeout occurs. rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running); multidata->sessions = still_running; assert(rc==CURLM_OK); } else if(timeout_ms == -1) //timeout_ms is -1 means we should delete the timer. { //call curl_multi_socket_action to update multidata->sessions, otherwise it will not be updated to 0 //when all transfers complete in some occasions(eg, GET, some objects hited while others miss). rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running); multidata->sessions = still_running; evtimer_del(&multidata->timer_event); } else //update the timer to the new value. { evtimer_add(&multidata->timer_event, &timeout); } return 0; //0-success; -1-error } //为每个Minio Host创建各自的curl multi handle struct doris_curl_multihd *doris_initialize_multihd_for_host(struct doris_http_instance *instance, struct dst_host_cnn_balance *host) { struct doris_curl_multihd *multidata; multidata = (struct doris_curl_multihd *)calloc(1, sizeof(struct doris_curl_multihd)); multidata->multi_hd = curl_multi_init(); multidata->evbase = instance->evbase; multidata->host = host; curl_multi_setopt(multidata->multi_hd, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX); curl_multi_setopt(multidata->multi_hd, CURLMOPT_MAX_HOST_CONNECTIONS, instance->param->maximum_host_cnns); curl_multi_setopt(multidata->multi_hd, CURLMOPT_MAX_PIPELINE_LENGTH, instance->param->maximum_pipelines); curl_multi_setopt(multidata->multi_hd, CURLMOPT_SOCKETFUNCTION, curl_socket_function_cb); curl_multi_setopt(multidata->multi_hd, CURLMOPT_SOCKETDATA, multidata); //curl_socket_function_cb *userp curl_multi_setopt(multidata->multi_hd, CURLMOPT_TIMERFUNCTION, curl_timer_function_cb); curl_multi_setopt(multidata->multi_hd, CURLMOPT_TIMERDATA, multidata); evtimer_assign(&multidata->timer_event, instance->evbase, libevent_timer_event_cb, multidata); return multidata; }