tango-tfe/platform/src/ssl_stream.cpp

/*
 *	SSL stream implementation.
 *	Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise.
 *	Author: zhengchao@iie.ac.cn
 *	Create: 2018-8-17
 */
#include <netinet/in.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <stdint.h>
#include <sys/socket.h>

#include <event2/event.h>
#include <event2/listener.h>
#include <event2/bufferevent.h>
#include <event2/bufferevent_ssl.h>
#include <event2/buffer.h>
#include <event2/thread.h>
#include <event2/dns.h>

#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <openssl/x509.h>
#include <openssl/x509v3.h>


#include <tfe_stream.h>
#include <tfe_utils.h>
#include <tfe_future.h>
#include <stream.h>
#include <key_keeper.h>
#include <ssl_sess_cache.h>
#include <ssl.h>
#include <MESA_htable.h>

#define SSL_EX_DATA_IDX_SSLMGR	0
#define MAX_NET_RETRIES 		50


struct ssl_mgr {
	int 			sslcomp;
	int 			no_ssl2;
	int 			no_ssl3;
	int 			no_tls10;
	int 			no_tls11;
	int 			no_tls12;
	CONST_SSL_METHOD * (*sslmethod) (void); 		//Parameter of SSL_CTX_new
	int 			sslversion;
	char			ssl_session_context[8];
	int 			cache_slot_num;
	int 			sess_expire_seconds;
	struct sess_cache * down_sess_cache;
	struct sess_cache * up_sess_cache;
	char			default_ciphers[TFE_STRING_MAX];
	DH *			dh;
	char *			ecdhcurve;
	char *			crlurl;
	uint8_t 		SSL_MODE_RELEASE_BUFFERS;
	void *			logger;
	char 			trust_CA_file[TFE_STRING_MAX];
	char 			trust_CA_dir[TFE_STRING_MAX];
	X509_STORE* 	trust_CA_store;
	struct key_keeper * keeper_of_keys;
};

struct __ssl_stream_debug
{
	evutil_socket_t fd;
};
struct ssl_stream {
	enum tfe_conn_dir dir;
	SSL *			ssl;
	struct ssl_mgr * mgr;
	union
	{
		struct ssl_chello * client_hello;	//dir=upstream, a little weird,  which send by downstream client.
		struct keyring*	keyring;			//dir=downstream.
	};
	struct __ssl_stream_debug _do_not_use;
};


struct ssl_chello {
	//client hello
	int 			version;
	char *			sni;
	char *			cipher_suites;
};


struct peek_client_hello_ctx {
	struct ssl_chello chello;
	unsigned char	sni_peek_retries;				/* max 64 SNI parse retries */
	struct event * ev;
	struct event_base * evbase;
	void *			logger;
};


struct ssl_connect_origin_ctx {
	struct bufferevent * bev;
	struct ssl_stream * s_stream;
	struct ssl_mgr * mgr;
	struct sockaddr addr;
	socklen_t		addrlen;
	void *			logger;

	evutil_socket_t fd_upstream;
	evutil_socket_t fd_downstream;
	struct event_base * evbase;
	struct future * f_peek_chello;
};


struct ask_keyring_ctx {
	int 			keyring_id;
	struct ssl_stream * origin_ssl;
	X509 *			origin_crt;
	int is_origin_crt_vaild;
	struct ssl_mgr * ssl_mgr;
	evutil_socket_t fd_downstream;
	struct event_base * evbase;
	
	struct future * f_query_cert;
	struct bufferevent * bev_down;
	struct ssl_stream* downstream;
};


/*
 * SSL shutdown context.
 */
struct ssl_shutdown_ctx {
	struct ssl_stream * s_stream;
	struct event_base * evbase;
	struct event * ev;
	unsigned int	retries;
};

struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir, struct ssl_chello * client_hello, struct keyring* crt)
{
	struct sockaddr addr;
	socklen_t addrlen;
	int ret=0;
	struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1);
	s_stream->dir=dir;
	s_stream->mgr=mgr;
	s_stream->_do_not_use.fd=fd;
	ret = getpeername(fd , &addr, &addrlen);
	assert(ret == 0);
	switch (dir)
	{
		case CONN_DIR_DOWNSTREAM:
			s_stream->ssl= downstream_ssl_create(mgr, crt);
			s_stream->keyring=crt;
			break;
		case CONN_DIR_UPSTREAM:
			s_stream->ssl= upstream_ssl_create(mgr, client_hello, fd);
			s_stream->client_hello=client_hello;
			break;
		default:
			assert(0);
	}
	return s_stream;
}


static void ssl_stream_free(struct ssl_stream * s_stream)
{
	SSL_free(s_stream->ssl);
	s_stream->ssl		= NULL;
	switch (s_stream->dir)
	{
		case CONN_DIR_DOWNSTREAM:
			if(s_stream->keyring!=NULL)
			{
				key_keeper_free_keyring(s_stream->keyring);
				s_stream->keyring=NULL;
			}
			break;
		case CONN_DIR_UPSTREAM:
			if(s_stream->client_hello!=NULL)
			{
				ssl_free_chello(s_stream->client_hello);
				s_stream->client_hello=NULL;
			}
			break;
		default:
			assert(0);
	}

	s_stream->mgr	= NULL;
	free(s_stream);
	return;
}


static int sslver_str2num(const char * version_str)
{
	int 			sslversion = -1;

	assert(OPENSSL_VERSION_NUMBER >= 0x10100000L);

	/*
	 * Support for SSLv2 and the corresponding SSLv2_method(),
	 * SSLv2_server_method() and SSLv2_client_method() functions were
	 * removed in OpenSSL 1.1.0.
	 */
	if (!strcmp(version_str, "ssl3")) 
	{
		sslversion			= SSL3_VERSION;
	}
	else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1"))
	{
		sslversion			= TLS1_VERSION;
	}
	else if (!strcmp(version_str, "tls11")) 
	{
		sslversion			= TLS1_1_VERSION;
	}
	else if (!strcmp(version_str, "tls12")) 
	{
		sslversion			= TLS1_2_VERSION;
	}
	else
	{
		sslversion			= -1;
	}

	return sslversion;
}


void ssl_manager_destroy(struct ssl_mgr * mgr)
{
	if(mgr->keeper_of_keys!=NULL)
	{
		key_keeper_destroy(mgr->keeper_of_keys);
	}
	if(mgr->trust_CA_store!=NULL)
	{
		X509_STORE_free(mgr->trust_CA_store);
		mgr->trust_CA_store=NULL;
	}
	free(mgr);
}


struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, void * logger) {
	struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1);
	int ret = 0, value = 0;
	char			version_str[TFE_SYMBOL_MAX];
	mgr->logger = logger;
	MESA_load_profile_string_def(ini_profile, section, "ssl_version", version_str, sizeof(version_str), "tls12");
	mgr->sslversion = sslver_str2num(version_str);

	if (mgr->sslversion < 0) {
		TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str);
		goto error_out;
	}


	//tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0.
	mgr->sslmethod = TLS_method;
	MESA_load_profile_int_def(ini_profile, section, "ssl_compression", & (mgr->sslcomp), 1);
	MESA_load_profile_int_def(ini_profile, section, "no_ssl2", & (mgr->no_ssl2), 1);
	MESA_load_profile_int_def(ini_profile, section, "no_ssl3", & (mgr->no_ssl3), 1);
	MESA_load_profile_int_def(ini_profile, section, "no_tls10", & (mgr->no_tls10), 1);
	MESA_load_profile_int_def(ini_profile, section, "no_tls11", & (mgr->no_tls11), 0);
	MESA_load_profile_int_def(ini_profile, section, "no_tls12", & (mgr->no_tls12), 0);
	MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", & (mgr->cache_slot_num), 
		4 * 1024 * 1024);
	MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", & (mgr->sess_expire_seconds), 30 * 60);

	mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM);
	mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM);


	
	mgr->keeper_of_keys = key_keeper_init(ini_profile, section, logger);

	if (mgr->keeper_of_keys == NULL)
	{
		TFE_LOG_ERROR(logger, "Certificate Manager initiate failed.");
		goto error_out;
	}
	mgr->trust_CA_store=X509_STORE_new();
	if(mgr->trust_CA_store==NULL)
	{
		TFE_LOG_ERROR(logger,"Failed at creating X509_STORE");
		goto error_out;
	}	
	ret=X509_STORE_set_default_paths(mgr->trust_CA_store);
	if(ret==0)
	{
		TFE_LOG_ERROR(logger,"Failed at setting default paths for X509_STORE");
		goto error_out;
	}
	MESA_load_profile_string_def(ini_profile, section, "trust_CA_file", mgr->trust_CA_file, sizeof(mgr->trust_CA_file), "");
	MESA_load_profile_string_def(ini_profile, section, "trust_CA_dir", mgr->trust_CA_dir, sizeof(mgr->trust_CA_dir), "");

	ret=X509_STORE_load_locations(mgr->trust_CA_store,strlen(mgr->trust_CA_file)>0? mgr->trust_CA_file:NULL
										,strlen(mgr->trust_CA_dir)>0? mgr->trust_CA_dir:NULL);
	if(ret==0)
	{
		TFE_LOG_ERROR(logger,"Failed at setting load locations for X509_STORE");
		goto error_out;
	}
	memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context));
	return mgr;

error_out:

	ssl_manager_destroy(mgr);
	return NULL;
}
int ssl_conn_verify_cert(X509_STORE *store, const SSL * ssl)
{
	int ret=0;
    STACK_OF(X509) * cert_chain = SSL_get_peer_cert_chain(ssl);
	if(cert_chain==NULL)
	{
		// The peer certificate chain is not necessarily available after reusing a session, in which case a NULL pointer is returned.
		return 1;
	}
	X509_STORE_CTX *ctx=X509_STORE_CTX_new();
	X509 * cert = sk_X509_value(cert_chain, 0);
	ret=X509_STORE_CTX_init(ctx, store, cert, cert_chain);
	assert(ret==1);
	
	//If a complete chain can be built and validated this function returns 1, otherwise it return zero or negtive code.
	ret=X509_verify_cert(ctx);
	X509_STORE_CTX_free(ctx);
	return (ret==1);
}


void peek_client_hello_ctx_free(void * ctx)
{
	struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)ctx;
	event_free(_ctx->ev);
	_ctx->ev			= NULL;
	free(_ctx->chello.sni);
	_ctx->chello.sni	= NULL;
	free(_ctx->chello.cipher_suites);
	_ctx->chello.cipher_suites = NULL;
	free(_ctx);
	return;
}


struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result)
{
	struct ssl_chello * p = (struct ssl_chello *)result, *copy = NULL;
	copy = ALLOC(struct ssl_chello, 1);

	if (p != NULL) {
		copy->sni			= tfe_strdup(p->sni);
		copy->cipher_suites = tfe_strdup(p->cipher_suites);
		copy->version		= p->version;
	}

	return copy;
} void ssl_free_chello(struct ssl_chello * p)
{
	if (p == NULL) {
		return;
	}

	free(p->sni);
	p->sni				= NULL;
	free(p->cipher_suites);
	p->cipher_suites	= NULL;
	free(p);
	return;
}


static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg)
{
	struct promise * promise = (struct promise *)arg;
	//use promise_get_ctx instead of promise_dettach_ctx for try more times.
	struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *)promise_get_ctx(promise);
	const char *	reason_too_many_retries = "too many tries";
	const char *	reason_see_no_client_hello = "see no client hello";
	const char *	reason = NULL;
	char *			sni = NULL;
	unsigned char	buf[1024];
	ssize_t 		n	= 0;
	const unsigned char * chello = NULL;
	int 			rv	= 0;

	n					= recv(fd, buf, sizeof(buf), MSG_PEEK);

	if (n == -1) {
		TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection\n");
		goto failed;
	}

	if (n == 0) {
		goto failed;
	}

	//todo: parse version and cipher suites.
	//or we should use sni proxy instead?  https://github.com/dlundquist/sniproxy/blob/master/src/tls.c
	rv					= ssl_tls_clienthello_parse(buf, n, 0, &chello, & (ctx->chello.sni));

	if (rv == 0) {
		promise_dettach_ctx(promise);
		promise_success(promise, & (ctx->chello));
		peek_client_hello_ctx_free(ctx);
	}
	else {
		if (!chello) {
			TFE_LOG_ERROR(ctx->logger, "Peeking did not yield a (truncated) ClientHello message, aborting connection\n");
			reason				= "see no client hello";
			goto failed;
		}

		if (ctx->sni_peek_retries++ > MAX_NET_RETRIES) {
			TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n");
			reason				= "too many peek retries";
			goto failed;
		}

		/* ssl_tls_clienthello_parse indicates that we
		* should retry later when we have more data, and we
		* haven't reached the maximum retry count yet.
		* Reschedule this event as timeout-only event in
		* order to prevent busy looping over the read event.
		* Because we only peeked at the pending bytes and
		* never actually read them, fd is still ready for
		* reading now.	We use 25 * 0.2 s = 5 s timeout. */
		struct timeval retry_delay = {
			0, 100
		};


		event_free(ctx->ev);
		ctx->ev 			= event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise);
		assert(ctx->ev != NULL);
		event_add(ctx->ev, &retry_delay);
	}

	return;

failed:
	promise_dettach_ctx(promise);
	promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason);
	peek_client_hello_ctx_free(ctx);
	return;
}


static void ssl_async_peek_client_hello(struct future * future, evutil_socket_t fd, struct event_base * evbase, 
	void * logger)
{
	struct event * ev = NULL;
	struct promise * p = future_to_promise(future);
	struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1);
	ctx->ev 			= event_new(evbase, fd, EV_READ, peek_client_hello_cb, p);
	ctx->logger 		= logger;
	event_add(evbase, NULL);
	promise_set_ctx(p, ctx, peek_client_hello_ctx_free);

	return;
}


/*
 * Create new SSL context for outgoing connections to the original destination.
 * If hostname sni is provided, use it for Server Name Indication.
 */
static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello,  evutil_socket_t fd)
{
	SSL_CTX *		sslctx = NULL;
	SSL *			ssl = NULL;
	SSL_SESSION *	sess = NULL;


	sslctx				= SSL_CTX_new(mgr->sslmethod());

	sslctx_set_opts(sslctx, mgr);

#if OPENSSL_VERSION_NUMBER			>= 0x10100000L

	if (mgr->sslversion) {
		if (SSL_CTX_set_min_proto_version(sslctx, chello->version) == 0 ||
			 SSL_CTX_set_max_proto_version(sslctx, chello->version) == 0) {
			SSL_CTX_free(sslctx);
			return NULL;
		}
	}

#endif /* OPENSSL_VERSION_NUMBER >= 0x10100000L */

	SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL);

	ssl 				= SSL_new(sslctx);
	SSL_CTX_free(sslctx);							/* SSL_new() increments refcount */

	if (!ssl) {
		return NULL;
	}

#ifndef OPENSSL_NO_TLSEXT

	if (chello->sni) {
		SSL_set_tlsext_host_name(ssl, sni);
	}

#endif /* !OPENSSL_NO_TLSEXT */

#ifdef SSL_MODE_RELEASE_BUFFERS

	/* lower memory footprint for idle connections */
	SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS);
#endif /* SSL_MODE_RELEASE_BUFFERS */

	struct sockaddr addr;
	socklen_t addrlen;
	int ret=0;
	ret = getpeername(fd , &addr, &addrlen);
	assert(ret==0);
	/* session resuming based on remote endpoint address and port */
	sess	= up_session_get(mgr->up_sess_cache, &addr, addrlen, chello->sni);	/* new sess insert */
	if (sess) 
	{
		SSL_set_session(ssl, sess); 				/* increments sess refcount */
		SSL_SESSION_free(sess);
	}
	return ssl;
}


void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx)
{
	if (ctx->s_stream != NULL) {
		ssl_stream_free(ctx->s_stream);
	}

	if (ctx->bev != NULL) {
		bufferevent_free(ctx->bev);
		ctx->bev			= NULL;
	}

	if (ctx->f_peek_chello != NULL) {
		future_destroy(ctx->f_peek_chello);
		ctx->f_peek_chello	= NULL;
	}

	free(ctx);
	return;
}


struct ssl_stream * ssl_conn_origin_result_release_stream(future_result_t * result) {
	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)result;
	struct ssl_stream * ret = ctx->s_stream;
	ctx->s_stream = NULL;							//giveup ownership
	return ret;
}


struct bufferevent * ssl_conn_origin_result_release_bev(future_result_t * result) {
	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)result;
	struct bufferevent * ret = ctx->bev;
	ctx->bev = NULL;								//giveup ownership
	return ret;

}

/*
 * Callback for meta events on the up- and downstream connection bufferevents.
 * Called when EOF has been reached, a connection has been made, and on errors.
 */
static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg)
{
	struct promise * promise = (struct promise *)arg;
	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)promise_dettach_ctx(promise);

	struct ssl_stream * s_stream = ctx->s_stream;
	SSL_SESSION *	ssl_sess = NULL;

	if (events & BEV_EVENT_ERROR) {
		promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to orignal server failed.");
	}
	else {
		if (events & BEV_EVENT_CONNECTED) {
			bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success
			ssl_sess			= SSL_get0_session(s_stream->ssl);
			up_session_set(s_stream->mgr->up_sess_cache, & (ctx->addr), ctx->addrlen, s_stream->client_hello.sni, 
				ssl_sess);
			promise_success(promise, ctx);
		}
		else {
			assert(0);
		}
	}

	ssl_connect_origin_ctx_free(ctx);
	return
}


static void peek_chello_on_succ(future_result_t * result, void * user)
{
	struct promise* p= (struct promise*)user;
	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)	promise_dettach_ctx(p);
	
	struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream.
	ctx->s_stream		= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL);
	ctx->bev			= bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_upstream, ctx->s_stream, 
		BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS);
	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1);
	bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, ctx);
	bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only
	future_destroy(ctx->f_peek_chello);
	ctx->f_peek_chello	= NULL;
	return;
}


static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user)
{
	struct promise* p= (struct promise*)user;
	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)	promise_dettach_ctx(p);

	promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream.");
	ssl_connect_origin_ctx_free(ctx);
	return;
}


extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, 
	evutil_socket_t fd_downstream, struct event_base * evbase)
{
	struct promise * p = future_to_promise(f);
	struct ssl_connect_origin_ctx * ctx = ALLOC(struct ssl_connect_origin_ctx, 1);
	int 			ret = 0;

	struct sockaddr addr;
	socklen_t		addrlen;

	ret 				= getpeername(fd_downstream, & (ctx->addr), & (ctx->addrlen));
	assert(ret == 0);
	ctx->fd_downstream	= fd_downstream;
	ctx->fd_upstream	= fd_upstream;
	ctx->evbase 		= evbase;
	ctx->mgr			= mgr;
	promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free);
	
	ctx->f_peek_chello	= future_create(peek_chello_on_succ, peek_chello_on_fail, p);
	ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger);
	return;
}


/*
 * Called by OpenSSL when a new src SSL session is created.
 * OpenSSL increments the refcount before calling the callback and will
 * decrement it again if we return 0.  Returning 1 will make OpenSSL skip
 * the refcount decrementing.  In other words, return 0 if we did not
 * keep a pointer to the object (which we never do here).
 */
static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess)
{
	struct ssl_mgr * mgr = (struct ssl_mgr *)
	SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr);

#ifdef HAVE_SSLV2

	/* Session resumption seems to fail for SSLv2 with protocol
	 * parsing errors, so we disable caching for SSLv2. */
	if (SSL_version(ssl) == SSL2_VERSION) {
		return 0;
	}

#endif /* HAVE_SSLV2 */

	if (sess) {
		down_session_set(mgr->down_sess_cache, sess);
	}

	return 0;
}


/*
 * Called by OpenSSL when a src SSL session should be removed.
 * OpenSSL calls SSL_SESSION_free() after calling the callback;
 * we do not need to free the reference here.
 */
static void ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess)
{
	struct ssl_mgr * mgr = (struct ssl_mgr *)
	SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr);
	assert(mgr != NULL);

	if (sess) {
		down_session_del(mgr->down_sess_cache, sess) ；
	}

	return;
}


/*
 * Called by OpenSSL when a src SSL session is requested by the client.
	OPENSSL_VERSION_NUMBER >= 0x10100000L required.
 */
static SSL_SESSION * ossl_sessget_cb(UNUSED SSL * ssl, const unsigned char * id, int idlen, int * copy)
{
	struct ssl_mgr * mgr = (struct ssl_mgr *)
	SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr);
	SSL_SESSION * sess;

	*copy				= 0;						/* SSL should not increment reference count of session */
	sess				= (SSL_SESSION *)
	down_session_get(mgr->down_sess_cache, id, idlen);
	return sess;
}


/*
 * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX.
 */
static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr)
{
	SSL_CTX_set_options(sslctx, SSL_OP_ALL);

#ifdef SSL_OP_TLS_ROLLBACK_BUG
	SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG);
#endif /* SSL_OP_TLS_ROLLBACK_BUG */

#ifdef SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION
	SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION);
#endif /* SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION */

#ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
	SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS);
#endif /* SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS */

#ifdef SSL_OP_NO_TICKET
	SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET);
#endif /* SSL_OP_NO_TICKET */

#ifdef SSL_OP_NO_SSLv2

#ifdef HAVE_SSLV2

	if (mgr->no_ssl2) {
#endif /* HAVE_SSLV2 */

		SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2);

#ifdef HAVE_SSLV2
	}

#endif /* HAVE_SSLV2 */

#endif /* !SSL_OP_NO_SSLv2 */

#ifdef HAVE_SSLV3

	if (mgr->no_ssl3) {
		SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3);
	}

#endif /* HAVE_SSLV3 */

#ifdef HAVE_TLSV10

	if (mgr->no_tls10) {
		SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1);
	}

#endif /* HAVE_TLSV10 */

#ifdef HAVE_TLSV11

	if (mgr->no_tls11) {
		SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1);
	}

#endif /* HAVE_TLSV11 */

#ifdef HAVE_TLSV12

	if (mgr->no_tls12) {
		SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2);
	}

#endif /* HAVE_TLSV12 */

#ifdef SSL_OP_NO_COMPRESSION

	if (!mgr->sslcomp) {
		SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION);
	}

#endif /* SSL_OP_NO_COMPRESSION */

	SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers);
}


/*
 * Create and set up a new SSL_CTX instance for terminating SSL.
 * Set up all the necessary callbacks, the keyring, the keyring chain and key.
 */
static SSL* downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt)
{
	SSL_CTX * sslctx	= SSL_CTX_new(mgr->sslmethod());
	if (!sslctx)
		return NULL;
	SSL* ssl=NULL;
	int ret=0;
	sslctx_set_opts(sslctx, mgr);

	//TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L
	if (mgr->sslversion) {
		if (SSL_CTX_set_min_proto_version(sslctx, mgr->sslversion) == 0 ||
			 SSL_CTX_set_max_proto_version(sslctx, mgr->sslversion) == 0) {
			SSL_CTX_free(sslctx);
			return NULL;
		}
	}

	SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb);
	SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb);
	SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb);
	SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL);
	SSL_CTX_set_session_id_context(sslctx, (void *) (mgr->ssl_session_context), 
		sizeof(mgr->ssl_session_context));

#ifndef OPENSSL_NO_DH

	if (mgr->dh) {
		SSL_CTX_set_tmp_dh(sslctx, mgr->dh);
	}
	else {
		SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback);
	}

#endif /* !OPENSSL_NO_DH */

#ifndef OPENSSL_NO_ECDH

	if (mgr->ecdhcurve) {
		EC_KEY * ecdh		= ssl_ec_by_name(mgr->ecdhcurve);
		SSL_CTX_set_tmp_ecdh(sslctx, ecdh);
		EC_KEY_free(ecdh);
	}
	else {
		EC_KEY * ecdh		= ssl_ec_by_name(NULL);
		SSL_CTX_set_tmp_ecdh(sslctx, ecdh);
		EC_KEY_free(ecdh);
	}

#endif /* !OPENSSL_NO_ECDH */

	SSL_CTX_use_certificate(sslctx, crt->cert);
	SSL_CTX_use_PrivateKey(sslctx, crt->key);

	for (int i = 0; i < sk_X509_num(crt->chain); i++) {
		X509 * c			= sk_X509_value(crt->chain, i);
		ssl_x509_refcount_inc(c);					/* next call consumes a reference */
		SSL_CTX_add_extra_chain_cert(sslctx, c);
	}
	ssl=SSL_new(sslctx);
	SSL_CTX_free(sslctx);			// SSL_new() increments refcount
	sslctx=NULL;
	ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr);
	assert(ret == 1);
		
	if (mgr->SSL_MODE_RELEASE_BUFFERS == 1)
	{
		/* lower memory footprint for idle connections */
		SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS);
	}

	return ssl;
}


void query_cert_ctx_free(struct ask_keyring_ctx * ctx)
{
	X509_free(ctx->origin_crt);

	if (ctx->f_query_cert != NULL) {
		future_destroy(ctx->f_query_cert);
		ctx->f_query_cert 	= NULL;
	}
	if(ctx->bev_down!=NULL)
	{
		bufferevent_free(ctx->bev_down);
	}
	if(ctx->downstream!=NULL)
	{
		ssl_stream_free(ctx->downstream);
	}
	return;
}

struct ssl_stream*  ssl_downstream_create_result_release_stream(future_result_t* result)
{
	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)result;
	struct ssl_stream* ret=ctx->downstream;
	ctx->downstream=NULL;
	return ret;
}
struct bufferevent* ssl_downstream_create_result_release_bev(future_result_t* result)
{
	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)result;
	struct bufferevent* ret=ctx->bev_down;
	ctx->bev_down=NULL;
	return ret;
}

void ask_keyring_on_succ(void * result, void * user)
{
	struct promise* p=(struct promise*)user;
	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)promise_dettach_ctx(promise);
	
	struct ssl_stream * downstream = NULL;
	struct keyring* crt=NULL;
	
	struct ssl_mgr* mgr=ctx->ssl_mgr;
	
	future_destroy(ctx->f_query_cert);
	ctx->f_query_cert=NULL;
	
	crt=key_keeper_release_cert(result);

	ctx->downstream	= ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, crt);
	
	ctx->bev_down	= bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl, 
		BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS);
	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1);

	promise_success(p, ctx);
	key_keeper_free_keyring(crt);
	query_cert_ctx_free(ctx);
	return;
}


void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user)
{
	struct promise* p=(struct promise*)user;
	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)promise_dettach_ctx(promise);
	promise_failed(p, error, what);
	query_cert_ctx_free(ctx);
	return;
}


/*
 * Create a SSL stream for the incoming connection, based on the upstream.
 */
void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base * evbase)
{

	assert(upstream->dir == CONN_DIR_UPSTREAM);

	struct ask_keyring_ctx * ctx = ALLOC(struct ask_keyring_ctx, 1);
	ctx->keyring_id 	= keyring_id;
	ctx->ssl_mgr		= mgr;
	ctx->fd_downstream	= fd_downstream;
	ctx->evbase			= evbase;

	if(upstream!=NULL)
	{
		ctx->origin_ssl 	= upstream;
		ctx->origin_crt 	= SSL_get_peer_certificate(upstream->ssl);
	}
	
	struct promise* p	= future_to_promise(f);
	promise_set_ctx(p, ctx, query_cert_ctx_free);
	ctx->is_origin_crt_vaild=ssl_conn_verify_cert(mgr->trust_CA_store, upstream->ssl);
	
	ctx->f_query_cert	= future_create(ask_keyring_on_succ, ask_keyring_on_fail, p);
	//todo add a is_valid_cert flag to keyring manager query API.
	key_keeper_async_ask(ctx->f_query_cert, mgr->keeper_of_keys, keyring_id, ctx->origin_crt, ctx->is_origin_crt_vaild, evbase);
	return;
}


/*
 * Cleanly shut down an SSL socket.  Libevent currently has no support for
 * cleanly shutting down an SSL socket so we work around that by using a
 * low-level event.  This works for recent versions of OpenSSL.  OpenSSL
 * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE
 * may or may not work.
 */
static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase) {
	struct ssl_shutdown_ctx * ctx = ALLOC(struct ssl_shutdown_ctx, 1);
	ctx->evbase = evbase;
	ctx->s_stream = s_stream;
	ctx->ev = NULL;
	ctx->retries = 0;
	return ctx;
} 

static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx)
{
	free(ctx);
}


/*
 * The shutdown socket event handler.  This is either
 * scheduled as a timeout-only event, or as a fd read or
 * fd write event, depending on whether SSL_shutdown()
 * indicates it needs read or write on the socket.
 */
static void pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void * arg)
{
	struct ssl_shutdown_ctx * ctx = (struct ssl_shutdown_ctx *)	arg;

	struct timeval retry_delay = {
		0, 100
	};

	void * logger		= ctx->s_stream->mgr->logger;
	short want			= 0;
	int rv				= 0, sslerr = 0;

	if (ctx->ev) {
		event_free(ctx->ev);
		ctx->ev 			= NULL;
	}

	/*
	 * Use the new (post-2008) semantics for SSL_shutdown() on a
	 * non-blocking socket.  SSL_shutdown() returns -1 and WANT_READ
	 * if the other end's close notify was not received yet, and
	 * WANT_WRITE it could not write our own close notify.
	 *
	 * This is a good collection of recent and relevant documents:
	 * http://bugs.python.org/issue8108
	 */
	rv					= SSL_shutdown(ctx->s_stream->ssl);

	if (rv == 1)
		goto complete;

	if (rv != -1) {
		goto retry;
	}

	switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv)))
	{
		case SSL_ERROR_WANT_READ:
			want = EV_READ;
			goto retry;
		case SSL_ERROR_WANT_WRITE:
			want = EV_WRITE;
			goto retry;
		case SSL_ERROR_ZERO_RETURN:
			goto retry;
		case SSL_ERROR_SYSCALL:
		case SSL_ERROR_SSL:
			goto complete;
		default:
			TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() "
			"error %i. Closing fd.\n", sslerr);
			goto complete;
	}

	goto complete;

retry:

	if (ctx->retries++ >= MAX_NET_RETRIES) {
		TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: "
		"Max retries reached. Closing fd.\n");
		goto complete;
	}

	ctx->ev 			= event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx);

	if (ctx->ev) {
		event_add(ctx->ev, &retry_delay);
		return;
	}

	TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: "
	"Cannot create event. Closing fd.\n");


complete:
	ssl_stream_free(ctx->s_stream);
	evutil_closesocket(fd);
	ssl_shutdown_ctx_free(ctx);

}


/*
 * Cleanly shutdown an SSL session on file descriptor fd using low-level
 * file descriptor readiness events on event base evbase.
 * Guarantees that SSL and the corresponding SSL_CTX are freed and the
 * socket is closed, eventually, or in the case of fatal errors, immediately.
 */
void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, evutil_socket_t fd)
{
	struct ssl_shutdown_ctx * sslshutctx = NULL;
	sslshutctx			= ssl_shutdown_ctx_new(s_stream, evbase);
	pxy_ssl_shutdown_cb(fd, 0, sslshutctx);
	return;
}