1、增加证书校验；2、修改cert manager为key keeper。

2018-08-26 18:26:24 +08:00
parent cf07c632fc
commit 5bb6a9c965
5 changed files with 246 additions and 152 deletions
--- a/platform/src/ssl_stream.cpp
+++ b/platform/src/ssl_stream.cpp
@@ -32,7 +32,7 @@
 #include <tfe_utils.h>
 #include <tfe_future.h>
 #include <stream.h>
-#include <cert.h>
+#include <key_keeper.h>
 #include <ssl_sess_cache.h>
 #include <ssl.h>
 #include <MESA_htable.h>
@@ -55,22 +55,32 @@ struct ssl_mgr {
 	int 			sess_expire_seconds;
 	struct sess_cache * down_sess_cache;
 	struct sess_cache * up_sess_cache;
-	char *			default_ciphers[TFE_SYMBOL_MAX];
+	char			default_ciphers[TFE_STRING_MAX];
 	DH *			dh;
 	char *			ecdhcurve;
 	char *			crlurl;
 	uint8_t 		SSL_MODE_RELEASE_BUFFERS;
 	void *			logger;
-	struct cert_mgr * cert_mgr;
+	char 			trust_CA_file[TFE_STRING_MAX];
+	char 			trust_CA_dir[TFE_STRING_MAX];
+	X509_STORE* 	trust_CA_store;
+	struct key_keeper * keeper_of_keys;
 };

-
+struct __ssl_stream_debug
+{
+	evutil_socket_t fd;
+};
 struct ssl_stream {
 	enum tfe_conn_dir dir;
 	SSL *			ssl;
 	struct ssl_mgr * mgr;
-	struct ssl_chello * client_hello;
-	struct ssl_stream* peer;
+	union
+	{
+		struct ssl_chello * client_hello;	//dir=upstream, a little weird,  which send by downstream client.
+		struct keyring*	keyring;			//dir=downstream.
+	};
+	struct __ssl_stream_debug _do_not_use;
 };


@@ -106,10 +116,11 @@ struct ssl_connect_origin_ctx {
 };


-struct query_cert_ctx {
+struct ask_keyring_ctx {
 	int 			keyring_id;
 	struct ssl_stream * origin_ssl;
 	X509 *			origin_crt;
+	int is_origin_crt_vaild;
 	struct ssl_mgr * ssl_mgr;
 	evutil_socket_t fd_downstream;
 	struct event_base * evbase;
@@ -130,23 +141,56 @@ struct ssl_shutdown_ctx {
 	unsigned int	retries;
 };

-
-struct ssl_stream * ssl_stream_create(struct ssl_chello * client_hello, struct ssl_mgr * mgr, struct sockaddr * addr, 
-	socklen_t addrlen) {
+struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir, struct ssl_chello * client_hello, struct keyring* crt)
+{
+	struct sockaddr addr;
+	socklen_t addrlen;
+	int ret=0;
 	struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1);
-	s_stream->client_hello = client_hello;
-	s_stream->ssl = upstream_ssl_create(mgr, client_hello, addr, addrlen);
+	s_stream->dir=dir;
+	s_stream->mgr=mgr;
+	s_stream->_do_not_use.fd=fd;
+	ret = getpeername(fd , &addr, &addrlen);
+	assert(ret == 0);
+	switch (dir)
+	{
+		case CONN_DIR_DOWNSTREAM:
+			s_stream->ssl= downstream_ssl_create(mgr, crt);
+			s_stream->keyring=crt;
+			break;
+		case CONN_DIR_UPSTREAM:
+			s_stream->ssl= upstream_ssl_create(mgr, client_hello, fd);
+			s_stream->client_hello=client_hello;
+			break;
+		default:
+			assert(0);
+	}
 	return s_stream;
-} 
+}


 static void ssl_stream_free(struct ssl_stream * s_stream)
 {
 	SSL_free(s_stream->ssl);
 	s_stream->ssl		= NULL;
-
-	if (s_stream->client_hello != NULL) {
-		ssl_free_chello(s_stream->client_hello;);
+	switch (s_stream->dir)
+	{
+		case CONN_DIR_DOWNSTREAM:
+			if(s_stream->keyring!=NULL)
+			{
+				key_keeper_free_keyring(s_stream->keyring);
+				s_stream->keyring=NULL;
+			}
+			break;
+		case CONN_DIR_UPSTREAM:
+			if(s_stream->client_hello!=NULL)
+			{
+				ssl_free_chello(s_stream->client_hello);
+				s_stream->client_hello=NULL;
+			}
+			break;
+		default:
+			assert(0);
 	}

 	s_stream->mgr	= NULL;
@@ -166,19 +210,24 @@ static int sslver_str2num(const char * version_str)
 	 * SSLv2_server_method() and SSLv2_client_method() functions were
 	 * removed in OpenSSL 1.1.0.
 	 */
-	if (!strcmp(version_str, "ssl3")) {
+	if (!strcmp(version_str, "ssl3")) 
+	{
 		sslversion			= SSL3_VERSION;
 	}
-	else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1")) {
+	else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1"))
+	{
 		sslversion			= TLS1_VERSION;
 	}
-	else if (!strcmp(version_str, "tls11")) {
+	else if (!strcmp(version_str, "tls11")) 
+	{
 		sslversion			= TLS1_1_VERSION;
 	}
-	else if (!strcmp(version_str, "tls12")) {
+	else if (!strcmp(version_str, "tls12")) 
+	{
 		sslversion			= TLS1_2_VERSION;
 	}
-	else {
+	else
+	{
 		sslversion			= -1;
 	}

@@ -188,6 +237,15 @@ static int sslver_str2num(const char * version_str)

 void ssl_manager_destroy(struct ssl_mgr * mgr)
 {
+	if(mgr->keeper_of_keys!=NULL)
+	{
+		key_keeper_destroy(mgr->keeper_of_keys);
+	}
+	if(mgr->trust_CA_store!=NULL)
+	{
+		X509_STORE_free(mgr->trust_CA_store);
+		mgr->trust_CA_store=NULL;
+	}
 	free(mgr);
 }

@@ -220,34 +278,70 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section

 	mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM);
 	mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM);
-	mgr->cert_mgr = cert_mgr_init(ini_profile, section);

-	if (mgr->cert_mgr == NULL) {
+
+	
+	mgr->keeper_of_keys = key_keeper_init(ini_profile, section, logger);
+
+	if (mgr->keeper_of_keys == NULL)
+	{
 		TFE_LOG_ERROR(logger, "Certificate Manager initiate failed.");
 		goto error_out;
 	}
+	mgr->trust_CA_store=X509_STORE_new();
+	if(mgr->trust_CA_store==NULL)
+	{
+		TFE_LOG_ERROR(logger,"Failed at creating X509_STORE");
+		goto error_out;
+	}	
+	ret=X509_STORE_set_default_paths(mgr->trust_CA_store);
+	if(ret==0)
+	{
+		TFE_LOG_ERROR(logger,"Failed at setting default paths for X509_STORE");
+		goto error_out;
+	}
+	MESA_load_profile_string_def(ini_profile, section, "trust_CA_file", mgr->trust_CA_file, sizeof(mgr->trust_CA_file), "");
+	MESA_load_profile_string_def(ini_profile, section, "trust_CA_dir", mgr->trust_CA_dir, sizeof(mgr->trust_CA_dir), "");

-
+	ret=X509_STORE_load_locations(mgr->trust_CA_store,strlen(mgr->trust_CA_file)>0? mgr->trust_CA_file:NULL
+										,strlen(mgr->trust_CA_dir)>0? mgr->trust_CA_dir:NULL);
+	if(ret==0)
+	{
+		TFE_LOG_ERROR(logger,"Failed at setting load locations for X509_STORE");
+		goto error_out;
+	}
 	memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context));
 	return mgr;

-
-
-
-
-
-
-
-
-
 error_out:

 	ssl_manager_destroy(mgr);
 	return NULL;
-} void peek_client_hello_ctx_free(void * ctx)
+}
+int ssl_conn_verify_cert(X509_STORE *store, const SSL * ssl)
 {
-	struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)
-	ctx;
+	int ret=0;
+    STACK_OF(X509) * cert_chain = SSL_get_peer_cert_chain(ssl);
+	if(cert_chain==NULL)
+	{
+		// The peer certificate chain is not necessarily available after reusing a session, in which case a NULL pointer is returned.
+		return 1;
+	}
+	X509_STORE_CTX *ctx=X509_STORE_CTX_new();
+	X509 * cert = sk_X509_value(cert_chain, 0);
+	ret=X509_STORE_CTX_init(ctx, store, cert, cert_chain);
+	assert(ret==1);
+	
+	//If a complete chain can be built and validated this function returns 1, otherwise it return zero or negtive code.
+	ret=X509_verify_cert(ctx);
+	X509_STORE_CTX_free(ctx);
+	return (ret==1);
+}
+
+
+void peek_client_hello_ctx_free(void * ctx)
+{
+	struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)ctx;
 	event_free(_ctx->ev);
 	_ctx->ev			= NULL;
 	free(_ctx->chello.sni);
@@ -259,10 +353,10 @@ error_out:
 }


-struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) {
-	struct ssl_chello * p = (struct ssl_chello *)
-	result, *copy = NULL;
-	copy = ALLOC(struct ssl_chello *, 1);
+struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result)
+{
+	struct ssl_chello * p = (struct ssl_chello *)result, *copy = NULL;
+	copy = ALLOC(struct ssl_chello, 1);

 	if (p != NULL) {
 		copy->sni			= tfe_strdup(p->sni);
@@ -270,7 +364,6 @@ struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) {
 		copy->version		= p->version;
 	}

-
 	return copy;
 } void ssl_free_chello(struct ssl_chello * p)
 {
@@ -290,6 +383,7 @@ struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) {
 static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg)
 {
 	struct promise * promise = (struct promise *)arg;
+	//use promise_get_ctx instead of promise_dettach_ctx for try more times.
 	struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *)promise_get_ctx(promise);
 	const char *	reason_too_many_retries = "too many tries";
 	const char *	reason_see_no_client_hello = "see no client hello";
@@ -358,7 +452,6 @@ failed:
 	promise_dettach_ctx(promise);
 	promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason);
 	peek_client_hello_ctx_free(ctx);
-	ssl_free_chello(result);
 	return;
 }

@@ -382,8 +475,7 @@ static void ssl_async_peek_client_hello(struct future * future, evutil_socket_t
 * Create new SSL context for outgoing connections to the original destination.
 * If hostname sni is provided, use it for Server Name Indication.
 */
-static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, struct sockaddr * addr, 
-	socklen_t addrlen)
+static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello,  evutil_socket_t fd)
 {
 	SSL_CTX *		sslctx = NULL;
 	SSL *			ssl = NULL;
@@ -429,17 +521,18 @@ static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello *
 	SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS);
 #endif /* SSL_MODE_RELEASE_BUFFERS */

-	//Todo: Refactor below session resumption.
-
+	struct sockaddr addr;
+	socklen_t addrlen;
+	int ret=0;
+	ret = getpeername(fd , &addr, &addrlen);
+	assert(ret==0);
 	/* session resuming based on remote endpoint address and port */
-	sess				= up_session_get(mgr->up_sess_cache, addr, 
-		addrlen, chello->sni);						/* new sess inst */
-
-	if (sess) {
+	sess	= up_session_get(mgr->up_sess_cache, &addr, addrlen, chello->sni);	/* new sess insert */
+	if (sess) 
+	{
 		SSL_set_session(ssl, sess); 				/* increments sess refcount */
 		SSL_SESSION_free(sess);
 	}
-
 	return ssl;
 }

@@ -520,7 +613,7 @@ static void peek_chello_on_succ(future_result_t * result, void * user)
 	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)	promise_dettach_ctx(p);
 	
 	struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream.
-	ctx->s_stream		= ssl_stream_create(chello, ctx->mgr, & (ctx->addr), ctx->addrlen);
+	ctx->s_stream		= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL);
 	ctx->bev			= bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_upstream, ctx->s_stream, 
 		BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS);
 	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1);
@@ -718,15 +811,15 @@ static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr)

 /*
 * Create and set up a new SSL_CTX instance for terminating SSL.
- * Set up all the necessary callbacks, the cert, the cert chain and key.
+ * Set up all the necessary callbacks, the keyring, the keyring chain and key.
 */
-static SSL_CTX * down_sslctx_create(struct ssl_mgr * mgr, struct cert * crt)
+static SSL* downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt)
 {
 	SSL_CTX * sslctx	= SSL_CTX_new(mgr->sslmethod());
-
 	if (!sslctx)
 		return NULL;
-
+	SSL* ssl=NULL;
+	int ret=0;
 	sslctx_set_opts(sslctx, mgr);

 	//TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L
@@ -771,7 +864,7 @@ static SSL_CTX * down_sslctx_create(struct ssl_mgr * mgr, struct cert * crt)

 #endif /* !OPENSSL_NO_ECDH */

-	SSL_CTX_use_certificate(sslctx, crt->crt);
+	SSL_CTX_use_certificate(sslctx, crt->cert);
 	SSL_CTX_use_PrivateKey(sslctx, crt->key);

 	for (int i = 0; i < sk_X509_num(crt->chain); i++) {
@@ -779,12 +872,23 @@ static SSL_CTX * down_sslctx_create(struct ssl_mgr * mgr, struct cert * crt)
 		ssl_x509_refcount_inc(c);					/* next call consumes a reference */
 		SSL_CTX_add_extra_chain_cert(sslctx, c);
 	}
+	ssl=SSL_new(sslctx);
+	SSL_CTX_free(sslctx);			// SSL_new() increments refcount
+	sslctx=NULL;
+	ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr);
+	assert(ret == 1);
+		
+	if (mgr->SSL_MODE_RELEASE_BUFFERS == 1)
+	{
+		/* lower memory footprint for idle connections */
+		SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS);
+	}

-	return sslctx;
+	return ssl;
 }


-void query_cert_ctx_free(struct query_cert_ctx * ctx)
+void query_cert_ctx_free(struct ask_keyring_ctx * ctx)
 {
 	X509_free(ctx->origin_crt);

@@ -805,67 +909,51 @@ void query_cert_ctx_free(struct query_cert_ctx * ctx)

 struct ssl_stream*  ssl_downstream_create_result_release_stream(future_result_t* result)
 {
-	struct query_cert_ctx * ctx = (struct query_cert_ctx *)result;
+	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)result;
 	struct ssl_stream* ret=ctx->downstream;
 	ctx->downstream=NULL;
 	return ret;
 }
 struct bufferevent* ssl_downstream_create_result_release_bev(future_result_t* result)
 {
-	struct query_cert_ctx * ctx = (struct query_cert_ctx *)result;
+	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)result;
 	struct bufferevent* ret=ctx->bev_down;
 	ctx->bev_down=NULL;
 	return ret;
 }

-void query_cert_on_succ(void * result, void * user)
+void ask_keyring_on_succ(void * result, void * user)
 {
 	struct promise* p=(struct promise*)user;
-	struct query_cert_ctx * ctx = (struct query_cert_ctx *)promise_dettach_ctx(promise);
+	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)promise_dettach_ctx(promise);
 	
 	struct ssl_stream * downstream = NULL;
-	struct cert* crt=NULL;
-	SSL_CTX * sslctx=NULL;
+	struct keyring* crt=NULL;
 	
 	struct ssl_mgr* mgr=ctx->ssl_mgr;
 	
 	future_destroy(ctx->f_query_cert);
 	ctx->f_query_cert=NULL;
 	
-	crt=cert_mgr_query_result_release_cert(result);
-	sslctx	= down_sslctx_create(mgr, crt);
-	assert(sslctx!=NULL);
-	
-	downstream			= ALLOC(struct ssl_stream, 1);
-	downstream->mgr 	= mgr;
-	downstream->dir 	= CONN_DIR_DOWNSTREAM;
-	downstream->ssl		= SSL_new(sslctx);
-	downstream->peer	= ctx->origin_ssl;
-	ctx->origin_ssl->peer= downstream;
-	int ret 			= SSL_set_ex_data(downstream->ssl, SSL_EX_DATA_IDX_SSLMGR, mgr);
-	assert(ret == 1);
-	SSL_CTX_free(sslctx);							/* SSL_new() increments refcount */
+	crt=key_keeper_release_cert(result);

-	if (mgr->SSL_MODE_RELEASE_BUFFERS == 1) {
-		/* lower memory footprint for idle connections */
-		SSL_set_mode(downstream->ssl, SSL_get_mode(downstream->ssl) | SSL_MODE_RELEASE_BUFFERS);
-	}
-	ctx->downstream	= downstream;
+	ctx->downstream	= ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, crt);
+	
 	ctx->bev_down	= bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl, 
 		BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS);
 	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1);

 	promise_success(p, ctx);
-	cert_mgr_free_cert(crt);
+	key_keeper_free_keyring(crt);
 	query_cert_ctx_free(ctx);
 	return;
 }


-void query_cert_on_fail(enum e_future_error error, const char * what, void * user)
+void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user)
 {
 	struct promise* p=(struct promise*)user;
-	struct query_cert_ctx * ctx = (struct query_cert_ctx *)promise_dettach_ctx(promise);
+	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)promise_dettach_ctx(promise);
 	promise_failed(p, error, what);
 	query_cert_ctx_free(ctx);
 	return;
@@ -873,26 +961,32 @@ void query_cert_on_fail(enum e_future_error error, const char * what, void * use


 /*
- * Create a SSL stream for the incoming connection, based on the a upstream.
+ * Create a SSL stream for the incoming connection, based on the upstream.
 */
 void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base * evbase)
 {

 	assert(upstream->dir == CONN_DIR_UPSTREAM);

-	struct query_cert_ctx * ctx = ALLOC(struct query_cert_ctx, 1);
+	struct ask_keyring_ctx * ctx = ALLOC(struct ask_keyring_ctx, 1);
 	ctx->keyring_id 	= keyring_id;
-	ctx->origin_ssl 	= upstream;
-	ctx->origin_crt 	= SSL_get_peer_certificate(upstream->ssl);
 	ctx->ssl_mgr		= mgr;
 	ctx->fd_downstream	= fd_downstream;
 	ctx->evbase			= evbase;
+
+	if(upstream!=NULL)
+	{
+		ctx->origin_ssl 	= upstream;
+		ctx->origin_crt 	= SSL_get_peer_certificate(upstream->ssl);
+	}
 	
 	struct promise* p	= future_to_promise(f);
 	promise_set_ctx(p, ctx, query_cert_ctx_free);
-	//todo: verify original certificate.
-	ctx->f_query_cert	= future_create(query_cert_on_succ, query_cert_on_fail, p);
-	cert_mgr_async_query(ctx->f_query_cert, mgr->cert_mgr, keyring_id, ctx->origin_crt, evbase);
+	ctx->is_origin_crt_vaild=ssl_conn_verify_cert(mgr->trust_CA_store, upstream->ssl);
+	
+	ctx->f_query_cert	= future_create(ask_keyring_on_succ, ask_keyring_on_fail, p);
+	//todo add a is_valid_cert flag to keyring manager query API.
+	key_keeper_async_ask(ctx->f_query_cert, mgr->keeper_of_keys, keyring_id, ctx->origin_crt, ctx->is_origin_crt_vaild, evbase);
 	return;
 }

@@ -904,17 +998,16 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct
 * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE
 * may or may not work.
 */
-static
-
-
-struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase) {
+static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase) {
 	struct ssl_shutdown_ctx * ctx = ALLOC(struct ssl_shutdown_ctx, 1);
 	ctx->evbase = evbase;
 	ctx->s_stream = s_stream;
 	ctx->ev = NULL;
 	ctx->retries = 0;
 	return ctx;
-} static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx)
+} 
+
+static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx)
 {
 	free(ctx);
 }