增加HTTP Post上传配置接口，支持主从双机备份与同步（冷备）

2021-08-25 18:40:20 +08:00
parent 67bafbefc9
commit 1aca701f12
28 changed files with 3295 additions and 648 deletions
--- a/server/doris_server_main.cpp
+++ b/server/doris_server_main.cpp
@@ -7,17 +7,24 @@
 #include <errno.h>
 #include <pthread.h>
 #include <string.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <net/if.h>

 #include <event2/thread.h>

 #include <MESA/MESA_prof_load.h>

-#include "doris_client.h"
+#include "doris_consumer_client.h"
+#include "doris_producer_client.h"
 #include "doris_server_main.h"
 #include "doris_server_http.h"

 struct doris_global_info g_doris_server_info;
-static unsigned long doris_vesion_20210804=20210804L;
+static unsigned long doris_version_20210825=20210825L;

 int doris_mkdir_according_path(const char * path)
 {
@@ -54,6 +61,79 @@ int doris_mkdir_according_path(const char * path)
 	return 0;
 }

+int32_t get_ip_by_ifname(const char *ifname)
+{
+	int sockfd;
+	struct ifreq ifr;
+	int32_t ip;
+
+	sockfd = socket(AF_INET, SOCK_DGRAM, 0);
+	if (-1 == sockfd) 
+		return INADDR_NONE;
+
+	strcpy(ifr.ifr_name,ifname); 
+	if (ioctl(sockfd, SIOCGIFADDR, &ifr) < 0) 
+	{
+		close(sockfd);
+		return INADDR_NONE;
+	}
+
+	ip = ((struct sockaddr_in*)&(ifr.ifr_addr))->sin_addr.s_addr;
+	close(sockfd);
+	return ip;
+}
+
+static inline void set_sockopt_keepalive(int sd, int keepidle, int keepintvl, int keepcnt)
+{
+	int keepalive = 1;
+	setsockopt(sd, SOL_SOCKET, SO_KEEPALIVE, (void*)&keepalive, sizeof(keepalive));
+	setsockopt(sd, SOL_TCP, TCP_KEEPIDLE, (void*)&keepidle, sizeof(keepidle));
+	setsockopt(sd, SOL_TCP, TCP_KEEPINTVL, (void*)&keepintvl, sizeof(keepintvl));
+	setsockopt(sd, SOL_TCP, TCP_KEEPCNT, (void*)&keepcnt, sizeof(keepcnt));
+}
+
+static inline void set_listen_sockopt(int sd)
+{
+	if(g_doris_server_info.sock_recv_bufsize > 0)
+	{
+		setsockopt(sd, SOL_SOCKET, SO_RCVBUF, &g_doris_server_info.sock_recv_bufsize, sizeof(u_int32_t));
+	}
+}
+
+int doris_create_listen_socket(int bind_port)
+{  
+	evutil_socket_t listener;
+	struct sockaddr_in sin;  
+	
+	listener = socket(AF_INET, SOCK_STREAM, 0);  
+	if(listener < 0)  
+	{  
+		MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "create socket error!\n");  
+		return -1;
+	}
+	set_sockopt_keepalive(listener, 300, 10, 2);
+	set_listen_sockopt(listener);
+	evutil_make_listen_socket_reuseable(listener);
+
+	sin.sin_family = AF_INET;  
+	sin.sin_addr.s_addr=htonl(INADDR_ANY);
+	sin.sin_port = htons(bind_port);  
+
+	if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0)
+	{  
+		printf("bind socket to port: %d error: %s!\n", bind_port, strerror(errno));  
+		MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "bind socket to port: %d error: %s!\n", bind_port, strerror(errno));  
+		assert(0);return -2;
+	}
+	if (listen(listener, 1024)<0)  
+	{  
+		MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "listen socket 1024 error!\n");  
+		return -3;
+	}
+	evutil_make_socket_nonblocking(listener);
+	return listener;
+}
+
 static int doris_chech_name_valid(const char *name)
 {
 	size_t i, namelen=strlen(name);
@@ -100,8 +180,11 @@ int32_t doris_read_profile_configs(const char *config_file)
 		assert(0);return -1;
 	}
 	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "cache_file_frag_size", &g_doris_server_info.cache_frag_size, 67108864);
-	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "doris_server_role_on", &g_doris_server_info.server_role_sw, 1);
+	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "http_post_max_frag_size", &g_doris_server_info.max_http_body_size, 67108864);
 	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "index_file_format_maat", &g_doris_server_info.idx_file_maat, 0);
+	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "http_post_max_concurrence", &g_doris_server_info.max_concurrent_reqs, 100000);
+	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "http_post_cluster_on", &g_doris_server_info.cluster_sync_mode, 0);
+	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "http_post_op_expires", &g_doris_server_info.post_vernode_ttl, 1200);

 	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "scan_index_file_interval", &g_doris_server_info.scan_idx_interval, 10);
 	MESA_load_profile_uint_def(config_file, "DORIS_SERVER", "https_connection_on", &g_doris_server_info.ssl_conn_on, 0);
@@ -122,18 +205,19 @@ int32_t doris_read_profile_configs(const char *config_file)
 	MESA_load_profile_int_def(config_file, "DORIS_SERVER", "fsstat_log_dst_port", &g_doris_server_info.fsstat_dst_port, 8125);

 	//LIBEVENT
-	MESA_load_profile_int_def(config_file, "DORIS_SERVER", "server_listen_port", &g_doris_server_info.server_port, 9898);
-	MESA_load_profile_int_def(config_file, "DORIS_SERVER", "manage_listen_port", &g_doris_server_info.manager_port, 2233);
+	MESA_load_profile_int_def(config_file, "DORIS_SERVER", "consumer_listen_port", &g_doris_server_info.consumer_port, 0);
+	MESA_load_profile_int_def(config_file, "DORIS_SERVER", "manager_listen_port", &g_doris_server_info.manager_port, 2233);
 	MESA_load_profile_int_def(config_file, "DORIS_SERVER", "socket_recv_bufsize", &g_doris_server_info.sock_recv_bufsize, 524288);
+	pthread_mutex_init(&g_doris_server_info.mutex_lock, NULL);
 	return 0;
 }

 static int doris_server_register_field_stat(struct doris_global_info *param)
 {
 	const char *field_names[DRS_FSSTAT_FIELD_NUM]={"RecvErrVer", "FileStarts", "FileComplete", "ClientInvReq",
-		"ClientMetaReq", "SendNoNewMeta", "ClientFileReq", "SendBytes", "SendFile404"};
+		"ClientMetaReq", "SendNoNewMeta", "ClientFileReq", "SendFiles", "SendBytes", "SendFile404", "VerExpire"};
 	const char *status_names[DRS_FSSTAT_STATUS_NUM]={"MemoryUsed"};
-	const char *column_names[DRS_FSSTAT_CLUMN_NUM]={"RecvFullVer", "RecvIncVer", "RecvFiles", "SendResMeta", "SendFiles",
+	const char *column_names[DRS_FSSTAT_CLUMN_NUM]={"RecvFullVer", "RecvIncVer", "RecvFiles", "PostOnWay", "SendResMeta",
 		"CurFullVer", "CurIncVer", "TotalCfgNum"};
 	int value;

@@ -188,7 +272,8 @@ static int32_t doris_init_config_for_business(struct doris_global_info *info, st
 {
 	char tmpbuffer[4096], tmp_dir[256], tmp_dir2[256], *bizname, *save=NULL;
 	struct doris_business *business;
-	map<string, struct doris_parameter *>::iterator iter;
+	map<string, struct doris_csum_param *>::iterator iter_csm;
+	map<string, struct doris_prod_param *>::iterator iter_prd;

 	if(0>=MESA_load_profile_string_nodef(config_file, "DORIS_SERVER", "business_system_list", tmpbuffer, sizeof(tmpbuffer)))
 	{
@@ -214,7 +299,7 @@ static int32_t doris_init_config_for_business(struct doris_global_info *info, st
 		}
 		info->name2business->insert(make_pair(string(business->bizname), business));

-		MESA_load_profile_uint_def(config_file, business->bizname, "grafana_monitor_status_id", &business->mm_status_codeid, 3);
+		MESA_load_profile_uint_def(config_file, business->bizname, "grafana_monitor_status_id", &business->mmval_status_codeid, 3);
 		MESA_load_profile_uint_def(config_file, business->bizname, "mem_cache_max_versions", &business->cache_max_versions, 0);
 		if(0>MESA_load_profile_string_nodef(config_file, business->bizname, "store_config_path", business->store_path_root, sizeof(business->store_path_root)))
 		{
@@ -230,28 +315,8 @@ static int32_t doris_init_config_for_business(struct doris_global_info *info, st
 		}
 		
 		MESA_load_profile_uint_def(config_file, business->bizname, "receive_config_way", &business->recv_way, RECV_WAY_DRS_CLIENT);
-		if(business->recv_way == RECV_WAY_DRS_CLIENT)
-		{
-			if(0>=MESA_load_profile_string_nodef(config_file, business->bizname, "doris_client_confile", tmp_dir, sizeof(tmp_dir)))
-			{
-				MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]doris_client_confile not found!", config_file);
-				assert(0);return -1;
-			}
-			if((iter=info->confile2param->find(string(tmp_dir))) != info->confile2param->end())
-			{
-				business->param = iter->second;
-			}
-			else
-			{
-				business->param = doris_parameter_new(tmp_dir, manage_evbase, info->log_runtime);
-				if(business->param == NULL)
-				{
-					assert(0);return -2;
-				}
-				info->confile2param->insert(make_pair(string(tmp_dir), business->param));
-			}
-		}
-		else
+		assert(business->recv_way==RECV_WAY_IDX_FILE || business->recv_way==RECV_WAY_DRS_CLIENT || business->recv_way==RECV_WAY_HTTP_POST);
+		if(business->recv_way == RECV_WAY_IDX_FILE)
 		{
 			if(0>MESA_load_profile_string_nodef(config_file, business->bizname, "receive_config_path_full", business->recv_path_full, sizeof(business->recv_path_full)))
 			{
@@ -264,12 +329,52 @@ static int32_t doris_init_config_for_business(struct doris_global_info *info, st
 				assert(0);return -1;
 			}
 		}
+		else //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC>Ҳ<EFBFBD><D2B2>Ҫ
+		{
+			if(business->recv_way==RECV_WAY_DRS_CLIENT || g_doris_server_info.cluster_sync_mode)
+			{
+				if(0>=MESA_load_profile_string_nodef(config_file, business->bizname, "doris_client_confile", tmp_dir, sizeof(tmp_dir)))
+				{
+					MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]doris_client_confile not found!", config_file);
+					assert(0);return -1;
+				}
+				if((iter_csm=info->confile2csmparam->find(string(tmp_dir))) != info->confile2csmparam->end())
+				{
+					business->param_csum = iter_csm->second;
+				}
+				else
+				{
+					business->param_csum = doris_csum_parameter_new(tmp_dir, manage_evbase, info->log_runtime);
+					if(business->param_csum == NULL)
+					{
+						assert(0);return -2;
+					}
+					info->confile2csmparam->insert(make_pair(string(tmp_dir), business->param_csum));
+				}
+			}
+			if(business->recv_way == RECV_WAY_HTTP_POST) //<2F><>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬ<EFBFBD><CDAC>
+			{
+				MESA_load_profile_uint_def(config_file, business->bizname, "producer_concurrence_allowed", &business->concurrency_allowed, 0);
+				if(MESA_load_profile_int_nodef(config_file, business->bizname, "producer_listen_port", &business->producer_port)<0)
+				{
+					MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [%s]producer_listen_port not found!", config_file, business->bizname);
+					assert(0);return -1;
+				}
+				if(g_doris_server_info.cluster_sync_mode && 
+					NULL==(business->param_prod = doris_prod_parameter_new(tmp_dir, manage_evbase, info->log_runtime)))
+				{
+					assert(0);return -2;
+				}
+				g_doris_server_info.business_post_num++;
+				business->token2node = new map<string, struct version_list_node *>;
+			}
+		}
 		business->fs_lineid = FS_register(info->fsstat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, business->bizname);;

 		snprintf(tmp_dir, 512, "latest_cfg_version_%s", business->bizname);
-		business->mm_latest_ver = MESA_Monitor_register(info->monitor, tmp_dir, MONITOR_METRICS_GAUGE, "Latest doris config version.");
+		business->mmid_latest_ver = MESA_Monitor_register(info->monitor, tmp_dir, MONITOR_METRICS_GAUGE, "Latest doris config version.");
 		snprintf(tmp_dir, 512, "total_config_num_%s", business->bizname);
-		business->mm_total_cfgnum = MESA_Monitor_register(info->monitor, tmp_dir, MONITOR_METRICS_GAUGE, "Total config num from latest full version till now.");
+		business->mmid_total_cfgnum = MESA_Monitor_register(info->monitor, tmp_dir, MONITOR_METRICS_GAUGE, "Total config num from latest full version till now.");
 	}
 	return 0;
 }
@@ -302,14 +407,16 @@ int main(int argc, char **argv)
 	struct timeval tv;
 	struct event timer_statistic_fsstat;
 	struct evhttp *manager_http;
+	char netname[32];

+	memset(&g_doris_server_info, 0, sizeof(struct doris_global_info));
 	if(doris_read_profile_configs(NIRVANA_CONFIG_FILE) || doris_server_register_field_stat(&g_doris_server_info))
 	{
 		return -1;
 	}
 	evthread_use_pthreads();
 	g_doris_server_info.name2business = new map<string, struct doris_business*>;
-	g_doris_server_info.confile2param = new map<string, struct doris_parameter *>;
+	g_doris_server_info.confile2csmparam = new map<string, struct doris_csum_param *>;
 	manage_evbase = event_base_new();

 	/*Doris manager server*/
@@ -327,7 +434,7 @@ int main(int argc, char **argv)
 	evhttp_set_cb(manager_http, "/doris/statistic/status", manager_statistic_status_requests_cb, NULL);
 	evhttp_set_cb(manager_http, "/doris/statistic/threads", manager_statistic_threads_requests_cb, NULL);
 	evhttp_set_gencb(manager_http, manager_generic_requests_cb, NULL);
-	g_doris_server_info.monitor = MESA_Monitor_instance_evhttp_new(manager_http, doris_vesion_20210804);
+	g_doris_server_info.monitor = MESA_Monitor_instance_evhttp_new(manager_http, doris_version_20210825);
 	if(evhttp_accept_socket(manager_http, g_doris_server_info.manager))
 	{
 		printf("evhttp_accept_socket %d error!\n", g_doris_server_info.manager);  
@@ -340,6 +447,24 @@ int main(int argc, char **argv)
 	{
 		return -4;
 	}
+	if(g_doris_server_info.business_post_num > 0)
+	{		
+		if((MESA_load_profile_string_nodef(NIRVANA_CONFIG_FILE, "DORIS_SERVER", "local_net_name", netname, sizeof(netname))<0) ||
+			(g_doris_server_info.local_ip = get_ip_by_ifname(netname))<0)
+		{
+			printf("%s: [DORIS_SERVER]local_net_name not valid", NIRVANA_CONFIG_FILE);
+			MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "%s: [DORIS_SERVER]local_net_name not valid", NIRVANA_CONFIG_FILE);
+			assert(0);return -11;
+		}
+		g_doris_server_info.mmid_post_server = MESA_Monitor_register(g_doris_server_info.monitor,
+			"http_post_server_status", MONITOR_METRICS_GAUGE, "Running status of doris http post server.");
+		MESA_Monitor_operation(g_doris_server_info.monitor, g_doris_server_info.mmid_post_server, MONITOR_VALUE_SET, PROMETHUES_POST_SERVER_DOWN);
+	}
+
+	if(g_doris_server_info.ssl_conn_on && NULL==(g_doris_server_info.ssl_instance=doris_connections_create_ssl_ctx()))
+	{
+		assert(0);return -8;
+	}

 	pthread_attr_init(&attr);
 	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
@@ -353,6 +478,14 @@ int main(int argc, char **argv)
 				assert(0);return -5;
 			}
 		}
+		else if(g_doris_server_info.business[i].recv_way == RECV_WAY_HTTP_POST)
+		{
+			if(pthread_create(&thread_desc, &attr, thread_http_post_recv_cfg, &g_doris_server_info.business[i]))
+			{
+				MESA_RUNTIME_LOGV3(g_doris_server_info.log_runtime, RLOG_LV_FATAL, "pthread_create(): %s", strerror(errno));
+				assert(0);return -6;
+			}
+		}
 		else
 		{
 			if(pthread_create(&thread_desc, &attr, thread_index_file_recv_cfg, &g_doris_server_info.business[i]))
@@ -364,17 +497,13 @@ int main(int argc, char **argv)
 	}

 	/*Doris http server*/
-	if(g_doris_server_info.server_role_sw)
+	if(g_doris_server_info.consumer_port)
 	{
-		g_doris_server_info.listener = doris_create_listen_socket(g_doris_server_info.server_port);
-		if(g_doris_server_info.listener < 0)
+		g_doris_server_info.listener_csum = doris_create_listen_socket(g_doris_server_info.consumer_port);
+		if(g_doris_server_info.listener_csum < 0)
 		{
 			return -7;
 		}
-		if(g_doris_server_info.ssl_conn_on && NULL==(g_doris_server_info.ssl_instance=doris_connections_create_ssl_ctx()))
-		{
-			assert(0);return -8;
-		}
 		for(u_int32_t i=0; i<g_doris_server_info.iothreads; i++)
 		{
 			if(pthread_create(&thread_desc, &attr, thread_doris_http_server, NULL))
@@ -396,4 +525,3 @@ int main(int argc, char **argv)
 	return 0;
 }

-