375 lines
11 KiB
C++
375 lines
11 KiB
C++
#include <sys/types.h>
|
||
#include <sys/stat.h>
|
||
#include <sys/ioctl.h>
|
||
#include <sys/socket.h>
|
||
#include <netinet/in.h>
|
||
#include <netinet/tcp.h>
|
||
#include <arpa/inet.h>
|
||
#include <net/if.h>
|
||
#include <unistd.h>
|
||
#include <stdio.h>
|
||
#include <stdlib.h>
|
||
#include <assert.h>
|
||
#include <errno.h>
|
||
#include <sys/time.h>
|
||
#include <time.h>
|
||
#include <string.h>
|
||
#include <openssl/sha.h>
|
||
#include <openssl/md5.h>
|
||
|
||
#include <MESA/MESA_prof_load.h>
|
||
|
||
#include "doris_client_http.h"
|
||
|
||
int32_t param_get_connected_hosts(struct doris_http_parameter *param)
|
||
{
|
||
return param->connected_hosts;
|
||
}
|
||
int32_t param_get_failed_hosts(struct doris_http_parameter *param)
|
||
{
|
||
return param->failed_hosts;
|
||
}
|
||
|
||
static int _unfold_IP_range(char* ip_range, char***ip_list, int size)
|
||
{
|
||
int i=0,count=0, ret=0;
|
||
int range_digits[5];
|
||
memset(range_digits,0,sizeof(range_digits));
|
||
ret=sscanf(ip_range,"%d.%d.%d.%d-%d",&range_digits[0],&range_digits[1],&range_digits[2],&range_digits[3],&range_digits[4]);
|
||
if(ret!=4&&ret!=5)
|
||
{
|
||
return 0;
|
||
}
|
||
if(ret==4&&range_digits[4]==0)
|
||
{
|
||
range_digits[4]=range_digits[3];
|
||
}
|
||
for(i=0;i<5;i++)
|
||
{
|
||
if(range_digits[i]<0||range_digits[i]>255)
|
||
{
|
||
return 0;
|
||
}
|
||
}
|
||
count=range_digits[4]-range_digits[3]+1;
|
||
*ip_list=(char**)realloc(*ip_list, sizeof(char*)*(size+count));
|
||
for(i=0;i<count;i++)
|
||
{
|
||
(*ip_list)[size+i]=(char*)malloc(64);
|
||
snprintf((*ip_list)[size+i],64,"%d.%d.%d.%d",range_digits[0],range_digits[1],range_digits[2],range_digits[3]+i);
|
||
}
|
||
return count;
|
||
}
|
||
|
||
static int unfold_IP_range(const char* ip_range, char***ip_list)
|
||
{
|
||
char *token=NULL,*sub_token=NULL,*saveptr;
|
||
char *buffer=(char*)calloc(sizeof(char),strlen(ip_range)+1);
|
||
int count=0;
|
||
strcpy(buffer,ip_range);
|
||
for (token = buffer; ; token= NULL)
|
||
{
|
||
sub_token= strtok_r(token,";", &saveptr);
|
||
if (sub_token == NULL)
|
||
break;
|
||
count+=_unfold_IP_range(sub_token, ip_list,count);
|
||
}
|
||
free(buffer);
|
||
return count;
|
||
}
|
||
|
||
static int decode_one_specific_group_ip(const char *iplist, struct dst_ipaddr_group *dstaddr)
|
||
{
|
||
int i, ipaddr_num;
|
||
char **balance_iplist=NULL;
|
||
|
||
ipaddr_num = unfold_IP_range(iplist, &balance_iplist);
|
||
if(ipaddr_num == 0)
|
||
{
|
||
free(dstaddr);
|
||
return -1;
|
||
}
|
||
dstaddr->dstaddr_num = ipaddr_num;
|
||
dstaddr->dstaddrs = (u_int32_t *)calloc(1, sizeof(u_int32_t)*ipaddr_num);
|
||
for(i=0; i<ipaddr_num; i++)
|
||
{
|
||
if(inet_pton(AF_INET, balance_iplist[i], (void *)&dstaddr->dstaddrs[i]) != 1)
|
||
{
|
||
free(dstaddr->dstaddrs);
|
||
return -1;
|
||
}
|
||
free(balance_iplist[i]);
|
||
}
|
||
free(balance_iplist);
|
||
return 0;
|
||
}
|
||
|
||
static int32_t load_and_init_server_group(char *dst_ipaddr, struct dst_ipaddr_group *dstaddrs, void *runtime_log)
|
||
{
|
||
u_int32_t intval;
|
||
|
||
intval = strlen(dst_ipaddr);
|
||
if(dst_ipaddr[intval-1] != ';')
|
||
{
|
||
dst_ipaddr[intval] = ';';
|
||
dst_ipaddr[intval+1] = '\0';
|
||
}
|
||
if(decode_one_specific_group_ip(dst_ipaddr, dstaddrs) < 0)
|
||
{
|
||
return -2;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
static void conhash_delay_destroy_timer_cb(int fd, short kind, void *userp)
|
||
{
|
||
struct time_event *delay_event=(struct time_event *)userp;
|
||
|
||
conhash_instance_free((struct consistent_hash *)delay_event->data);
|
||
free(delay_event);
|
||
}
|
||
|
||
static void load_balance_common_timer_start(struct event *time_event)
|
||
{
|
||
struct timeval tv;
|
||
|
||
tv.tv_sec = 2;
|
||
tv.tv_usec = 0;
|
||
evtimer_add(time_event, &tv);
|
||
}
|
||
|
||
static void conhash_handle_delay_destroy(struct event_base *evbase, struct consistent_hash *conhash)
|
||
{
|
||
struct time_event *delay_event;
|
||
|
||
delay_event = (struct time_event *)malloc(sizeof(struct time_event));
|
||
delay_event->data = conhash;
|
||
evtimer_assign(&delay_event->timer_event, evbase, conhash_delay_destroy_timer_cb, delay_event);
|
||
load_balance_common_timer_start(&delay_event->timer_event);
|
||
}
|
||
|
||
static void conhash_insert_dest_host(struct dst_host_cnn_balance *balance)
|
||
{
|
||
struct conhash_bucket bucket;
|
||
struct consistent_hash *tmphash, *newhash=NULL;
|
||
enum CONHASH_ERRCODE code;
|
||
|
||
bucket.bucket_id = balance->dstip;
|
||
bucket.point_num = DEFAULT_HOST_CAPACITY * LOAD_BALANC_VIRT_TIMES;;
|
||
bucket.tag = NULL;
|
||
|
||
newhash = conhash_instance_copy(balance->param->conhash);
|
||
code = conhash_insert_bucket(newhash, &bucket);
|
||
assert(code == CONHASH_OK);
|
||
|
||
tmphash = balance->param->conhash;
|
||
balance->param->conhash = newhash;
|
||
conhash_handle_delay_destroy(balance->param->evbase, tmphash);
|
||
}
|
||
|
||
static void conhash_remove_dest_host(struct dst_host_cnn_balance *balance)
|
||
{
|
||
struct consistent_hash *tmphash, *newhash=NULL;
|
||
enum CONHASH_ERRCODE code;
|
||
|
||
newhash = conhash_instance_copy(balance->param->conhash);
|
||
code = conhash_remove_bucket(newhash, balance->dstip, NULL);
|
||
assert(code == CONHASH_OK || code==CONHASH_BUCKET_NOT_FOUND);
|
||
|
||
tmphash = balance->param->conhash;
|
||
balance->param->conhash = newhash;
|
||
conhash_handle_delay_destroy(balance->param->evbase, tmphash);
|
||
}
|
||
|
||
static void client_bufferevent_error_cb(struct bufferevent *bev, short event, void *arg)
|
||
{
|
||
struct dst_host_cnn_balance *balance = (struct dst_host_cnn_balance *)arg;
|
||
const char *errtype;
|
||
|
||
if(event & BEV_EVENT_CONNECTED)
|
||
{
|
||
errtype = "connected";
|
||
balance->connection_status = TCP_STATUS_CONNECTED;
|
||
balance->param->connected_hosts += 1; //<2F><><EFBFBD>й<EFBFBD><D0B9><EFBFBD><EFBFBD>߳<EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>̽<EFBFBD><CCBD>
|
||
if(balance->connect_failed)
|
||
{
|
||
balance->connect_failed = false;
|
||
balance->param->failed_hosts -= 1;
|
||
}
|
||
conhash_insert_dest_host(balance);
|
||
assert(balance->param->connected_hosts > 0);
|
||
assert(balance->param->failed_hosts >= 0);
|
||
}
|
||
else
|
||
{
|
||
if (event & BEV_EVENT_TIMEOUT) {
|
||
errtype = "Timed out";
|
||
}
|
||
else if (event & BEV_EVENT_EOF) {
|
||
errtype = "disconnected";
|
||
}
|
||
else if (event & BEV_EVENT_ERROR) {
|
||
errtype = "some other error";
|
||
}
|
||
else {
|
||
errtype = "unkonwn error";
|
||
}
|
||
bufferevent_free(bev);
|
||
balance->bev = NULL;
|
||
|
||
if(balance->connection_status == TCP_STATUS_CONNECTED)
|
||
{
|
||
balance->param->connected_hosts -= 1;
|
||
}
|
||
if(!balance->connect_failed)
|
||
{
|
||
balance->connect_failed = true;
|
||
balance->param->failed_hosts += 1;
|
||
}
|
||
balance->connection_status = TCP_STATUS_DISCONNECT;
|
||
|
||
load_balance_common_timer_start(&balance->timer_detect);
|
||
conhash_remove_dest_host(balance);
|
||
|
||
assert(balance->param->connected_hosts >= 0);
|
||
assert(balance->param->failed_hosts > 0);
|
||
}
|
||
|
||
MESA_HANDLE_RUNTIME_LOGV2(balance->param->runtime_log, RLOG_LV_INFO, "connection event: %s, addr: %s", errtype, balance->srvaddr);
|
||
}
|
||
|
||
int do_bufferevent_connection(struct doris_http_parameter *param, struct dst_host_cnn_balance *balance, struct sockaddr *server_addr)
|
||
{
|
||
if(balance->connection_status==TCP_STATUS_CONNECTING)
|
||
{
|
||
return 0;
|
||
}
|
||
|
||
if(NULL == (balance->bev = bufferevent_socket_new(param->evbase, -1, BEV_OPT_CLOSE_ON_FREE)))
|
||
{
|
||
assert(0);return -1;
|
||
}
|
||
|
||
if(bufferevent_socket_connect(balance->bev, server_addr, sizeof(struct sockaddr_in)))
|
||
{
|
||
bufferevent_free(balance->bev);
|
||
balance->bev = NULL;
|
||
MESA_HANDLE_RUNTIME_LOGV2(param->runtime_log, RLOG_LV_FATAL, "bufferevent_socket_connect error: %s", strerror(errno));
|
||
assert(0);return -2;
|
||
}
|
||
balance->connection_status = TCP_STATUS_CONNECTING;
|
||
bufferevent_setcb(balance->bev, NULL, NULL, client_bufferevent_error_cb, balance);
|
||
bufferevent_setwatermark(balance->bev, EV_WRITE, 100*1024*1024UL, 0);
|
||
bufferevent_enable(balance->bev, EV_READ|EV_WRITE|EV_PERSIST);
|
||
return 0;
|
||
}
|
||
|
||
void param_connection_detect_timer_cb(int fd, short kind, void *userp)
|
||
{
|
||
struct dst_host_cnn_balance *balance = (struct dst_host_cnn_balance *)userp;
|
||
|
||
if(do_bufferevent_connection(balance->param, balance, (struct sockaddr*)&balance->addr))
|
||
{
|
||
assert(0);
|
||
}
|
||
}
|
||
|
||
static int32_t doris_launch_group_connection(struct doris_http_parameter *param, struct event_base* evbase)
|
||
{
|
||
char ipaddr[64];
|
||
|
||
param->balance = (struct dst_host_cnn_balance *)calloc(1, sizeof(struct dst_host_cnn_balance)*param->ipgroup.dstaddr_num);
|
||
|
||
for(u_int32_t i=0; i<param->ipgroup.dstaddr_num; i++) //<2F><><EFBFBD><EFBFBD>
|
||
{
|
||
param->balance[i].dstip = param->ipgroup.dstaddrs[i];
|
||
param->balance[i].addr.sin_family = AF_INET;
|
||
param->balance[i].addr.sin_port = htons(param->manage_port);
|
||
param->balance[i].addr.sin_addr.s_addr = param->ipgroup.dstaddrs[i];
|
||
param->balance[i].connection_status = TCP_STATUS_IDLE;
|
||
param->balance[i].param = param;
|
||
|
||
inet_ntop(AF_INET, ¶m->ipgroup.dstaddrs[i], ipaddr, 64);
|
||
snprintf(param->balance[i].srvaddr, 64, "%s:%u", ipaddr, param->server_port);
|
||
|
||
evtimer_assign(¶m->balance[i].timer_detect, evbase, param_connection_detect_timer_cb, ¶m->balance[i]);
|
||
|
||
if(do_bufferevent_connection(param, ¶m->balance[i], (struct sockaddr *)¶m->balance[i].addr))
|
||
{
|
||
MESA_HANDLE_RUNTIME_LOGV2(param->runtime_log, RLOG_LV_FATAL, "do_bufferevent_connection error: %s.", strerror(errno));
|
||
return -1;
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
struct doris_http_parameter *doris_http_parameter_new(const char* profile_path, const char* section, struct event_base* evbase, void *runtime_log)
|
||
{
|
||
struct doris_http_parameter *param;
|
||
u_int32_t intval;
|
||
char dst_ipaddr[8192];
|
||
|
||
param = (struct doris_http_parameter *)calloc(1, sizeof(struct doris_http_parameter));
|
||
param->runtime_log = runtime_log;
|
||
param->evbase = evbase;
|
||
|
||
//multi curl
|
||
MESA_load_profile_uint_def(profile_path, section, "max_connection_per_host", &intval, 1);
|
||
param->maximum_host_cnns = intval;
|
||
MESA_load_profile_uint_def(profile_path, section, "max_cnnt_pipeline_num", &intval, 20);
|
||
param->maximum_pipelines = intval;
|
||
MESA_load_profile_uint_def(profile_path, section, "max_curl_transfer_timeout_s", &intval, 0);
|
||
param->transfer_timeout = intval;
|
||
MESA_load_profile_int_def(profile_path, section, "https_connection_on", ¶m->ssl_connection, 0);
|
||
MESA_load_profile_uint_def(profile_path, section, "max_curl_session_num", ¶m->max_http_sessions, 30);
|
||
|
||
//Server
|
||
if(MESA_load_profile_uint_def(profile_path, section, "http_server_listen_port", ¶m->server_port, 9898) < 0)
|
||
{
|
||
free(param);
|
||
MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] http_server_listen_port not found.", profile_path, section);
|
||
return NULL;
|
||
}
|
||
MESA_load_profile_uint_def(profile_path, section, "http_server_manage_port", ¶m->manage_port, 2233);
|
||
|
||
if(MESA_load_profile_string_nodef(profile_path, section, "http_server_ip_list", dst_ipaddr, 8192) < 0)
|
||
{
|
||
free(param);
|
||
MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] http_server_ip_list not found.", profile_path, section);
|
||
return NULL;
|
||
}
|
||
|
||
if(load_and_init_server_group(dst_ipaddr, ¶m->ipgroup, runtime_log))
|
||
{
|
||
MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Decode %s [%s] http_server_ip_list format error: %s", profile_path, section, dst_ipaddr);
|
||
assert(0);return NULL;
|
||
}
|
||
param->conhash = conhash_instance_new(NULL, 0);;
|
||
if(doris_launch_group_connection(param, evbase))
|
||
{
|
||
assert(0);return NULL;
|
||
}
|
||
return param;
|
||
}
|
||
|
||
struct doris_http_instance *doris_http_instance_new(struct doris_http_parameter *param, struct event_base* evbase, void *runtimelog)
|
||
{
|
||
struct doris_http_instance *instance;
|
||
struct doris_curl_multihd *multihd;
|
||
|
||
instance = (struct doris_http_instance *)calloc(1, sizeof(struct doris_http_instance));
|
||
instance->runtime_log = runtimelog;
|
||
instance->evbase = evbase;
|
||
instance->param = param;
|
||
|
||
//Ϊÿһ<C3BF><D2BB>minio IP<49><50><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>multi_handle
|
||
instance->server_hosts = new map<u_int32_t, doris_curl_multihd*>;
|
||
for(u_int32_t i=0; i<param->ipgroup.dstaddr_num; i++)
|
||
{
|
||
multihd = doris_initialize_multihd_for_host(instance, ¶m->balance[i]);
|
||
instance->server_hosts->insert(make_pair(param->ipgroup.dstaddrs[i], multihd));
|
||
}
|
||
return instance;
|
||
}
|
||
|