TSG-1518 改进TFE查询Certstore过程的异常处理机制

platform/src/key_keeper.cpp

@@ -6,6 +6,9 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
+#include <unistd.h>
+#include <sys/prctl.h>
+#include <arpa/inet.h>
 
 #include <MESA/MESA_prof_load.h>
 #include <MESA/MESA_htable.h>
@@ -46,7 +49,7 @@ struct key_keeper
 	struct key_keeper_stat stat;
 
 	int cert_expire_time;
-
+	pthread_t thread;
 };
 
 
@@ -67,6 +70,8 @@ struct key_keeper_promise_ctx
 	unsigned int key_len;
 };
 
+long long certstore_is_unavailable = 0;
+
 static void key_keeper_promise_free_ctx(void* ctx)
 {
 	struct key_keeper_promise_ctx* _ctx = (struct key_keeper_promise_ctx*)ctx;
@@ -462,6 +467,79 @@ struct evhttp_connection* key_keeper_evhttp_init(struct event_base * evbase, str
 	return evhttp_connection_base_new(evbase, dnsbase, cert_store_host, cert_store_port);
 }
 
+static int health_check(struct key_keeper *keeper)
+{
+	char req_buff[1024] = { 0 };
+	char rsp_buff[1024] = { 0 };
+	struct sockaddr_in addr;
+	char post_head[] =
+    	"POST %s HTTP/1.1\r\n"
+    	"Host: %s:%d\r\n"
+    	"User-Agent: curl/7.47.0\r\n"
+    	"Accept: */*\r\n"
+    	"Content-Length: %d\r\n\r\n"
+    	"%s\r\n";
+	sprintf(req_buff, post_head, "/ca?health_check=1", keeper->cert_store_host, keeper->cert_store_port, strlen("health check"), "health check");
+
+	int sockfd = socket(AF_INET, SOCK_STREAM, 0);
+    if (sockfd == -1)
+	{
+		TFE_LOG_ERROR(keeper->logger, "CertStore health check thread fail to create socket(), %s", strerror(errno));
+		goto error; 
+	}
+    memset(&addr, 0, sizeof(addr));
+    addr.sin_family = AF_INET;
+    addr.sin_port = htons(keeper->cert_store_port);
+    addr.sin_addr.s_addr = inet_addr(keeper->cert_store_host);
+    if (connect(sockfd, (const struct sockaddr *)&addr, sizeof(addr)) == -1)
+    {
+        TFE_LOG_ERROR(keeper->logger, "CertStore health check thread fail to connect(), %s", strerror(errno));
+		goto error; 
+    }
+	if (write(sockfd, req_buff, strlen(req_buff)) == -1)
+	{
+        TFE_LOG_ERROR(keeper->logger, "CertStore health check thread fail to write(), %s", strerror(errno));
+		goto error; 
+	}
+	if (read(sockfd, rsp_buff, sizeof(rsp_buff)) == -1)
+	{
+        TFE_LOG_ERROR(keeper->logger, "CertStore health check thread fail to read(), %s", strerror(errno));
+		goto error; 
+	}
+	close(sockfd);
+	return 0;
+
+error:
+	if (sockfd)
+		close(sockfd);
+	return -1;
+}
+static void * certstore_health_check_thread(void * arg)
+{
+    struct key_keeper * keeper = (struct key_keeper *)arg;
+	assert(keeper != NULL && keeper->thread == pthread_self());
+	TFE_LOG_INFO(keeper->logger, "CertStore health check thread is running...");
+	prctl(PR_SET_NAME, "CertStore_health_check");
+
+	while (1)
+	{
+		if (health_check(keeper) == -1)
+		{
+			ATOMIC_INC(&certstore_is_unavailable);
+			TFE_LOG_ERROR(keeper->logger, "CertStore health check thread on fail: certstore is unavailable !!!");
+		}
+		else
+		{
+			ATOMIC_ZERO(&certstore_is_unavailable);
+		}
+		sleep(1);
+	}
+
+	TFE_LOG_ERROR(keeper->logger, "CertStore health check thread is exit");
+
+	return NULL;
+}
+
 struct key_keeper* key_keeper_init(const char * profile, const char* section, void* logger)
 {
 	struct key_keeper* keeper = ALLOC(struct key_keeper, 1);
@@ -516,6 +594,20 @@ struct key_keeper* key_keeper_init(const char * profile, const char* section, vo
 			goto error_out;
 		}
 	}
+	// KK_MODE_CERT_STORE: create thread for certstore health check
+	else
+	{
+		pthread_attr_t attr;
+		pthread_attr_init(&attr);
+		pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+		int ret = pthread_create(&keeper->thread, NULL, certstore_health_check_thread, (void *)keeper);
+		if (unlikely(ret < 0))
+		{
+			TFE_LOG_ERROR(keeper->logger, "Failed at creating certstore health check thread: %s", strerror(errno));
+			goto error_out;
+		}
+	}
+	
 	TFE_LOG_INFO(logger, "MESA_load_profile, [%s]: mode:%s, no_cache:%u ,ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d, cert_expire_time:%d",
 		section, tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds, keeper->cert_expire_time);
 

platform/src/ssl_stream.cpp

@@ -64,6 +64,8 @@ static unsigned char SSL_ALPN_HTTP_2[]={2, 'h','2',0};
  */
 #define DFLT_CURVE "prime256v1"
 
+extern long long certstore_is_unavailable;
+
 enum ssl_stream_stat
 {
 	SSL_UP_NEW,
@@ -1420,6 +1422,13 @@ static void peek_chello_on_succ(future_result_t * result, void * user)
 	{
 		s_stream->up_parts.action=SSL_ACTION_INTERCEPT;
 	}
+
+	if (ATOMIC_READ(&certstore_is_unavailable) > 3)
+	{
+		s_stream->up_parts.action=SSL_ACTION_PASSTHROUGH;
+		TFE_LOG_ERROR(ctx->mgr->logger, "CertStore is unavailable, PASSTHROUGH");
+	}
+	
 	ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_INTERCEPT_STATE, s_stream->up_parts.action);
 	ctx->s_stream = s_stream;
 	if(s_stream->up_parts.action==SSL_ACTION_PASSTHROUGH)