/* * SSL stream implementation. * Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise. * Author: zhengchao@iie.ac.cn * Create: 2018-8-17 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SSL_EX_DATA_IDX_SSLMGR 0 #define MAX_NET_RETRIES 50 struct ssl_mgr { int sslcomp; int no_ssl2; int no_ssl3; int no_tls10; int no_tls11; int no_tls12; CONST_SSL_METHOD * (*sslmethod) (void); //Parameter of SSL_CTX_new int sslversion; char ssl_session_context[8]; int cache_slot_num; int sess_expire_seconds; struct sess_cache * down_sess_cache; struct sess_cache * up_sess_cache; char default_ciphers[TFE_STRING_MAX]; DH * dh; char * ecdhcurve; char * crlurl; uint8_t SSL_MODE_RELEASE_BUFFERS; void * logger; char trust_CA_file[TFE_STRING_MAX]; char trust_CA_dir[TFE_STRING_MAX]; X509_STORE* trust_CA_store; struct key_keeper * keeper_of_keys; }; struct __ssl_stream_debug { evutil_socket_t fd; }; struct ssl_stream { enum tfe_conn_dir dir; SSL * ssl; struct ssl_mgr * mgr; union { struct ssl_chello * client_hello; //dir=upstream, a little weird, which send by downstream client. struct keyring* keyring; //dir=downstream. }; struct __ssl_stream_debug _do_not_use; }; struct ssl_chello { //client hello int version; char * sni; char * cipher_suites; }; struct peek_client_hello_ctx { struct ssl_chello chello; unsigned char sni_peek_retries; /* max 64 SNI parse retries */ struct event * ev; struct event_base * evbase; void * logger; }; struct ssl_connect_origin_ctx { struct bufferevent * bev; struct ssl_stream * s_stream; struct ssl_mgr * mgr; struct sockaddr addr; socklen_t addrlen; void * logger; evutil_socket_t fd_upstream; evutil_socket_t fd_downstream; struct event_base * evbase; struct future * f_peek_chello; }; struct ask_keyring_ctx { int keyring_id; struct ssl_stream * origin_ssl; X509 * origin_crt; int is_origin_crt_vaild; struct ssl_mgr * ssl_mgr; evutil_socket_t fd_downstream; struct event_base * evbase; struct future * f_query_cert; struct bufferevent * bev_down; struct ssl_stream* downstream; }; /* * SSL shutdown context. */ struct ssl_shutdown_ctx { struct ssl_stream * s_stream; struct event_base * evbase; struct event * ev; unsigned int retries; }; struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir, struct ssl_chello * client_hello, struct keyring* crt) { struct sockaddr addr; socklen_t addrlen; int ret=0; struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1); s_stream->dir=dir; s_stream->mgr=mgr; s_stream->_do_not_use.fd=fd; ret = getpeername(fd , &addr, &addrlen); assert(ret == 0); switch (dir) { case CONN_DIR_DOWNSTREAM: s_stream->ssl= downstream_ssl_create(mgr, crt); s_stream->keyring=crt; break; case CONN_DIR_UPSTREAM: s_stream->ssl= upstream_ssl_create(mgr, client_hello, fd); s_stream->client_hello=client_hello; break; default: assert(0); } return s_stream; } static void ssl_stream_free(struct ssl_stream * s_stream) { SSL_free(s_stream->ssl); s_stream->ssl = NULL; switch (s_stream->dir) { case CONN_DIR_DOWNSTREAM: if(s_stream->keyring!=NULL) { key_keeper_free_keyring(s_stream->keyring); s_stream->keyring=NULL; } break; case CONN_DIR_UPSTREAM: if(s_stream->client_hello!=NULL) { ssl_free_chello(s_stream->client_hello); s_stream->client_hello=NULL; } break; default: assert(0); } s_stream->mgr = NULL; free(s_stream); return; } static int sslver_str2num(const char * version_str) { int sslversion = -1; assert(OPENSSL_VERSION_NUMBER >= 0x10100000L); /* * Support for SSLv2 and the corresponding SSLv2_method(), * SSLv2_server_method() and SSLv2_client_method() functions were * removed in OpenSSL 1.1.0. */ if (!strcmp(version_str, "ssl3")) { sslversion = SSL3_VERSION; } else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1")) { sslversion = TLS1_VERSION; } else if (!strcmp(version_str, "tls11")) { sslversion = TLS1_1_VERSION; } else if (!strcmp(version_str, "tls12")) { sslversion = TLS1_2_VERSION; } else { sslversion = -1; } return sslversion; } void ssl_manager_destroy(struct ssl_mgr * mgr) { if(mgr->keeper_of_keys!=NULL) { key_keeper_destroy(mgr->keeper_of_keys); } if(mgr->trust_CA_store!=NULL) { X509_STORE_free(mgr->trust_CA_store); mgr->trust_CA_store=NULL; } free(mgr); } struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, void * logger) { struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1); int ret = 0, value = 0; char version_str[TFE_SYMBOL_MAX]; mgr->logger = logger; MESA_load_profile_string_def(ini_profile, section, "ssl_version", version_str, sizeof(version_str), "tls12"); mgr->sslversion = sslver_str2num(version_str); if (mgr->sslversion < 0) { TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str); goto error_out; } //tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0. mgr->sslmethod = TLS_method; MESA_load_profile_int_def(ini_profile, section, "ssl_compression", & (mgr->sslcomp), 1); MESA_load_profile_int_def(ini_profile, section, "no_ssl2", & (mgr->no_ssl2), 1); MESA_load_profile_int_def(ini_profile, section, "no_ssl3", & (mgr->no_ssl3), 1); MESA_load_profile_int_def(ini_profile, section, "no_tls10", & (mgr->no_tls10), 1); MESA_load_profile_int_def(ini_profile, section, "no_tls11", & (mgr->no_tls11), 0); MESA_load_profile_int_def(ini_profile, section, "no_tls12", & (mgr->no_tls12), 0); MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", & (mgr->cache_slot_num), 4 * 1024 * 1024); MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", & (mgr->sess_expire_seconds), 30 * 60); mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); mgr->keeper_of_keys = key_keeper_init(ini_profile, section, logger); if (mgr->keeper_of_keys == NULL) { TFE_LOG_ERROR(logger, "Certificate Manager initiate failed."); goto error_out; } mgr->trust_CA_store=X509_STORE_new(); if(mgr->trust_CA_store==NULL) { TFE_LOG_ERROR(logger,"Failed at creating X509_STORE"); goto error_out; } ret=X509_STORE_set_default_paths(mgr->trust_CA_store); if(ret==0) { TFE_LOG_ERROR(logger,"Failed at setting default paths for X509_STORE"); goto error_out; } MESA_load_profile_string_def(ini_profile, section, "trust_CA_file", mgr->trust_CA_file, sizeof(mgr->trust_CA_file), ""); MESA_load_profile_string_def(ini_profile, section, "trust_CA_dir", mgr->trust_CA_dir, sizeof(mgr->trust_CA_dir), ""); ret=X509_STORE_load_locations(mgr->trust_CA_store,strlen(mgr->trust_CA_file)>0? mgr->trust_CA_file:NULL ,strlen(mgr->trust_CA_dir)>0? mgr->trust_CA_dir:NULL); if(ret==0) { TFE_LOG_ERROR(logger,"Failed at setting load locations for X509_STORE"); goto error_out; } memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context)); return mgr; error_out: ssl_manager_destroy(mgr); return NULL; } int ssl_conn_verify_cert(X509_STORE *store, const SSL * ssl) { int ret=0; STACK_OF(X509) * cert_chain = SSL_get_peer_cert_chain(ssl); if(cert_chain==NULL) { // The peer certificate chain is not necessarily available after reusing a session, in which case a NULL pointer is returned. return 1; } X509_STORE_CTX *ctx=X509_STORE_CTX_new(); X509 * cert = sk_X509_value(cert_chain, 0); ret=X509_STORE_CTX_init(ctx, store, cert, cert_chain); assert(ret==1); //If a complete chain can be built and validated this function returns 1, otherwise it return zero or negtive code. ret=X509_verify_cert(ctx); X509_STORE_CTX_free(ctx); return (ret==1); } void peek_client_hello_ctx_free(void * ctx) { struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)ctx; event_free(_ctx->ev); _ctx->ev = NULL; free(_ctx->chello.sni); _ctx->chello.sni = NULL; free(_ctx->chello.cipher_suites); _ctx->chello.cipher_suites = NULL; free(_ctx); return; } struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) { struct ssl_chello * p = (struct ssl_chello *)result, *copy = NULL; copy = ALLOC(struct ssl_chello, 1); if (p != NULL) { copy->sni = tfe_strdup(p->sni); copy->cipher_suites = tfe_strdup(p->cipher_suites); copy->version = p->version; } return copy; } void ssl_free_chello(struct ssl_chello * p) { if (p == NULL) { return; } free(p->sni); p->sni = NULL; free(p->cipher_suites); p->cipher_suites = NULL; free(p); return; } static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) { struct promise * promise = (struct promise *)arg; //use promise_get_ctx instead of promise_dettach_ctx for try more times. struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *)promise_get_ctx(promise); const char * reason_too_many_retries = "too many tries"; const char * reason_see_no_client_hello = "see no client hello"; const char * reason = NULL; char * sni = NULL; unsigned char buf[1024]; ssize_t n = 0; const unsigned char * chello = NULL; int rv = 0; n = recv(fd, buf, sizeof(buf), MSG_PEEK); if (n == -1) { TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection\n"); goto failed; } if (n == 0) { goto failed; } //todo: parse version and cipher suites. //or we should use sni proxy instead? https://github.com/dlundquist/sniproxy/blob/master/src/tls.c rv = ssl_tls_clienthello_parse(buf, n, 0, &chello, & (ctx->chello.sni)); if (rv == 0) { promise_dettach_ctx(promise); promise_success(promise, & (ctx->chello)); peek_client_hello_ctx_free(ctx); } else { if (!chello) { TFE_LOG_ERROR(ctx->logger, "Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); reason = "see no client hello"; goto failed; } if (ctx->sni_peek_retries++ > MAX_NET_RETRIES) { TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n"); reason = "too many peek retries"; goto failed; } /* ssl_tls_clienthello_parse indicates that we * should retry later when we have more data, and we * haven't reached the maximum retry count yet. * Reschedule this event as timeout-only event in * order to prevent busy looping over the read event. * Because we only peeked at the pending bytes and * never actually read them, fd is still ready for * reading now. We use 25 * 0.2 s = 5 s timeout. */ struct timeval retry_delay = { 0, 100 }; event_free(ctx->ev); ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); assert(ctx->ev != NULL); event_add(ctx->ev, &retry_delay); } return; failed: promise_dettach_ctx(promise); promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason); peek_client_hello_ctx_free(ctx); return; } static void ssl_async_peek_client_hello(struct future * future, evutil_socket_t fd, struct event_base * evbase, void * logger) { struct event * ev = NULL; struct promise * p = future_to_promise(future); struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1); ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); ctx->logger = logger; event_add(evbase, NULL); promise_set_ctx(p, ctx, peek_client_hello_ctx_free); return; } /* * Create new SSL context for outgoing connections to the original destination. * If hostname sni is provided, use it for Server Name Indication. */ static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, evutil_socket_t fd) { SSL_CTX * sslctx = NULL; SSL * ssl = NULL; SSL_SESSION * sess = NULL; sslctx = SSL_CTX_new(mgr->sslmethod()); sslctx_set_opts(sslctx, mgr); #if OPENSSL_VERSION_NUMBER >= 0x10100000L if (mgr->sslversion) { if (SSL_CTX_set_min_proto_version(sslctx, chello->version) == 0 || SSL_CTX_set_max_proto_version(sslctx, chello->version) == 0) { SSL_CTX_free(sslctx); return NULL; } } #endif /* OPENSSL_VERSION_NUMBER >= 0x10100000L */ SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); ssl = SSL_new(sslctx); SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ if (!ssl) { return NULL; } #ifndef OPENSSL_NO_TLSEXT if (chello->sni) { SSL_set_tlsext_host_name(ssl, sni); } #endif /* !OPENSSL_NO_TLSEXT */ #ifdef SSL_MODE_RELEASE_BUFFERS /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); #endif /* SSL_MODE_RELEASE_BUFFERS */ struct sockaddr addr; socklen_t addrlen; int ret=0; ret = getpeername(fd , &addr, &addrlen); assert(ret==0); /* session resuming based on remote endpoint address and port */ sess = up_session_get(mgr->up_sess_cache, &addr, addrlen, chello->sni); /* new sess insert */ if (sess) { SSL_set_session(ssl, sess); /* increments sess refcount */ SSL_SESSION_free(sess); } return ssl; } void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx) { if (ctx->s_stream != NULL) { ssl_stream_free(ctx->s_stream); } if (ctx->bev != NULL) { bufferevent_free(ctx->bev); ctx->bev = NULL; } if (ctx->f_peek_chello != NULL) { future_destroy(ctx->f_peek_chello); ctx->f_peek_chello = NULL; } free(ctx); return; } struct ssl_stream * ssl_conn_origin_result_release_stream(future_result_t * result) { struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)result; struct ssl_stream * ret = ctx->s_stream; ctx->s_stream = NULL; //giveup ownership return ret; } struct bufferevent * ssl_conn_origin_result_release_bev(future_result_t * result) { struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)result; struct bufferevent * ret = ctx->bev; ctx->bev = NULL; //giveup ownership return ret; } /* * Callback for meta events on the up- and downstream connection bufferevents. * Called when EOF has been reached, a connection has been made, and on errors. */ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg) { struct promise * promise = (struct promise *)arg; struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)promise_dettach_ctx(promise); struct ssl_stream * s_stream = ctx->s_stream; SSL_SESSION * ssl_sess = NULL; if (events & BEV_EVENT_ERROR) { promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to orignal server failed."); } else { if (events & BEV_EVENT_CONNECTED) { bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success ssl_sess = SSL_get0_session(s_stream->ssl); up_session_set(s_stream->mgr->up_sess_cache, & (ctx->addr), ctx->addrlen, s_stream->client_hello.sni, ssl_sess); promise_success(promise, ctx); } else { assert(0); } } ssl_connect_origin_ctx_free(ctx); return } static void peek_chello_on_succ(future_result_t * result, void * user) { struct promise* p= (struct promise*)user; struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. ctx->s_stream = ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL); ctx->bev = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_upstream, ctx->s_stream, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1); bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, ctx); bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only future_destroy(ctx->f_peek_chello); ctx->f_peek_chello = NULL; return; } static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user) { struct promise* p= (struct promise*)user; struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream."); ssl_connect_origin_ctx_free(ctx); return; } extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, evutil_socket_t fd_downstream, struct event_base * evbase) { struct promise * p = future_to_promise(f); struct ssl_connect_origin_ctx * ctx = ALLOC(struct ssl_connect_origin_ctx, 1); int ret = 0; struct sockaddr addr; socklen_t addrlen; ret = getpeername(fd_downstream, & (ctx->addr), & (ctx->addrlen)); assert(ret == 0); ctx->fd_downstream = fd_downstream; ctx->fd_upstream = fd_upstream; ctx->evbase = evbase; ctx->mgr = mgr; promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free); ctx->f_peek_chello = future_create(peek_chello_on_succ, peek_chello_on_fail, p); ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); return; } /* * Called by OpenSSL when a new src SSL session is created. * OpenSSL increments the refcount before calling the callback and will * decrement it again if we return 0. Returning 1 will make OpenSSL skip * the refcount decrementing. In other words, return 0 if we did not * keep a pointer to the object (which we never do here). */ static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); #ifdef HAVE_SSLV2 /* Session resumption seems to fail for SSLv2 with protocol * parsing errors, so we disable caching for SSLv2. */ if (SSL_version(ssl) == SSL2_VERSION) { return 0; } #endif /* HAVE_SSLV2 */ if (sess) { down_session_set(mgr->down_sess_cache, sess); } return 0; } /* * Called by OpenSSL when a src SSL session should be removed. * OpenSSL calls SSL_SESSION_free() after calling the callback; * we do not need to free the reference here. */ static void ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); assert(mgr != NULL); if (sess) { down_session_del(mgr->down_sess_cache, sess) ； } return; } /* * Called by OpenSSL when a src SSL session is requested by the client. OPENSSL_VERSION_NUMBER >= 0x10100000L required. */ static SSL_SESSION * ossl_sessget_cb(UNUSED SSL * ssl, const unsigned char * id, int idlen, int * copy) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); SSL_SESSION * sess; *copy = 0; /* SSL should not increment reference count of session */ sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache, id, idlen); return sess; } /* * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX. */ static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr) { SSL_CTX_set_options(sslctx, SSL_OP_ALL); #ifdef SSL_OP_TLS_ROLLBACK_BUG SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG); #endif /* SSL_OP_TLS_ROLLBACK_BUG */ #ifdef SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); #endif /* SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION */ #ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); #endif /* SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS */ #ifdef SSL_OP_NO_TICKET SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET); #endif /* SSL_OP_NO_TICKET */ #ifdef SSL_OP_NO_SSLv2 #ifdef HAVE_SSLV2 if (mgr->no_ssl2) { #endif /* HAVE_SSLV2 */ SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); #ifdef HAVE_SSLV2 } #endif /* HAVE_SSLV2 */ #endif /* !SSL_OP_NO_SSLv2 */ #ifdef HAVE_SSLV3 if (mgr->no_ssl3) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); } #endif /* HAVE_SSLV3 */ #ifdef HAVE_TLSV10 if (mgr->no_tls10) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1); } #endif /* HAVE_TLSV10 */ #ifdef HAVE_TLSV11 if (mgr->no_tls11) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1); } #endif /* HAVE_TLSV11 */ #ifdef HAVE_TLSV12 if (mgr->no_tls12) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2); } #endif /* HAVE_TLSV12 */ #ifdef SSL_OP_NO_COMPRESSION if (!mgr->sslcomp) { SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION); } #endif /* SSL_OP_NO_COMPRESSION */ SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); } /* * Create and set up a new SSL_CTX instance for terminating SSL. * Set up all the necessary callbacks, the keyring, the keyring chain and key. */ static SSL* downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt) { SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); if (!sslctx) return NULL; SSL* ssl=NULL; int ret=0; sslctx_set_opts(sslctx, mgr); //TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L if (mgr->sslversion) { if (SSL_CTX_set_min_proto_version(sslctx, mgr->sslversion) == 0 || SSL_CTX_set_max_proto_version(sslctx, mgr->sslversion) == 0) { SSL_CTX_free(sslctx); return NULL; } } SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); SSL_CTX_set_session_id_context(sslctx, (void *) (mgr->ssl_session_context), sizeof(mgr->ssl_session_context)); #ifndef OPENSSL_NO_DH if (mgr->dh) { SSL_CTX_set_tmp_dh(sslctx, mgr->dh); } else { SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); } #endif /* !OPENSSL_NO_DH */ #ifndef OPENSSL_NO_ECDH if (mgr->ecdhcurve) { EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } else { EC_KEY * ecdh = ssl_ec_by_name(NULL); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } #endif /* !OPENSSL_NO_ECDH */ SSL_CTX_use_certificate(sslctx, crt->cert); SSL_CTX_use_PrivateKey(sslctx, crt->key); for (int i = 0; i < sk_X509_num(crt->chain); i++) { X509 * c = sk_X509_value(crt->chain, i); ssl_x509_refcount_inc(c); /* next call consumes a reference */ SSL_CTX_add_extra_chain_cert(sslctx, c); } ssl=SSL_new(sslctx); SSL_CTX_free(sslctx); // SSL_new() increments refcount sslctx=NULL; ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); assert(ret == 1); if (mgr->SSL_MODE_RELEASE_BUFFERS == 1) { /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); } return ssl; } void query_cert_ctx_free(struct ask_keyring_ctx * ctx) { X509_free(ctx->origin_crt); if (ctx->f_query_cert != NULL) { future_destroy(ctx->f_query_cert); ctx->f_query_cert = NULL; } if(ctx->bev_down!=NULL) { bufferevent_free(ctx->bev_down); } if(ctx->downstream!=NULL) { ssl_stream_free(ctx->downstream); } return; } struct ssl_stream* ssl_downstream_create_result_release_stream(future_result_t* result) { struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)result; struct ssl_stream* ret=ctx->downstream; ctx->downstream=NULL; return ret; } struct bufferevent* ssl_downstream_create_result_release_bev(future_result_t* result) { struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)result; struct bufferevent* ret=ctx->bev_down; ctx->bev_down=NULL; return ret; } void ask_keyring_on_succ(void * result, void * user) { struct promise* p=(struct promise*)user; struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)promise_dettach_ctx(promise); struct ssl_stream * downstream = NULL; struct keyring* crt=NULL; struct ssl_mgr* mgr=ctx->ssl_mgr; future_destroy(ctx->f_query_cert); ctx->f_query_cert=NULL; crt=key_keeper_release_cert(result); ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, crt); ctx->bev_down = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); promise_success(p, ctx); key_keeper_free_keyring(crt); query_cert_ctx_free(ctx); return; } void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user) { struct promise* p=(struct promise*)user; struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)promise_dettach_ctx(promise); promise_failed(p, error, what); query_cert_ctx_free(ctx); return; } /* * Create a SSL stream for the incoming connection, based on the upstream. */ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base * evbase) { assert(upstream->dir == CONN_DIR_UPSTREAM); struct ask_keyring_ctx * ctx = ALLOC(struct ask_keyring_ctx, 1); ctx->keyring_id = keyring_id; ctx->ssl_mgr = mgr; ctx->fd_downstream = fd_downstream; ctx->evbase = evbase; if(upstream!=NULL) { ctx->origin_ssl = upstream; ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl); } struct promise* p = future_to_promise(f); promise_set_ctx(p, ctx, query_cert_ctx_free); ctx->is_origin_crt_vaild=ssl_conn_verify_cert(mgr->trust_CA_store, upstream->ssl); ctx->f_query_cert = future_create(ask_keyring_on_succ, ask_keyring_on_fail, p); //todo add a is_valid_cert flag to keyring manager query API. key_keeper_async_ask(ctx->f_query_cert, mgr->keeper_of_keys, keyring_id, ctx->origin_crt, ctx->is_origin_crt_vaild, evbase); return; } /* * Cleanly shut down an SSL socket. Libevent currently has no support for * cleanly shutting down an SSL socket so we work around that by using a * low-level event. This works for recent versions of OpenSSL. OpenSSL * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE * may or may not work. */ static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase) { struct ssl_shutdown_ctx * ctx = ALLOC(struct ssl_shutdown_ctx, 1); ctx->evbase = evbase; ctx->s_stream = s_stream; ctx->ev = NULL; ctx->retries = 0; return ctx; } static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx) { free(ctx); } /* * The shutdown socket event handler. This is either * scheduled as a timeout-only event, or as a fd read or * fd write event, depending on whether SSL_shutdown() * indicates it needs read or write on the socket. */ static void pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void * arg) { struct ssl_shutdown_ctx * ctx = (struct ssl_shutdown_ctx *) arg; struct timeval retry_delay = { 0, 100 }; void * logger = ctx->s_stream->mgr->logger; short want = 0; int rv = 0, sslerr = 0; if (ctx->ev) { event_free(ctx->ev); ctx->ev = NULL; } /* * Use the new (post-2008) semantics for SSL_shutdown() on a * non-blocking socket. SSL_shutdown() returns -1 and WANT_READ * if the other end's close notify was not received yet, and * WANT_WRITE it could not write our own close notify. * * This is a good collection of recent and relevant documents: * http://bugs.python.org/issue8108 */ rv = SSL_shutdown(ctx->s_stream->ssl); if (rv == 1) goto complete; if (rv != -1) { goto retry; } switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) { case SSL_ERROR_WANT_READ: want = EV_READ; goto retry; case SSL_ERROR_WANT_WRITE: want = EV_WRITE; goto retry; case SSL_ERROR_ZERO_RETURN: goto retry; case SSL_ERROR_SYSCALL: case SSL_ERROR_SSL: goto complete; default: TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() " "error %i. Closing fd.\n", sslerr); goto complete; } goto complete; retry: if (ctx->retries++ >= MAX_NET_RETRIES) { TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " "Max retries reached. Closing fd.\n"); goto complete; } ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); if (ctx->ev) { event_add(ctx->ev, &retry_delay); return; } TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " "Cannot create event. Closing fd.\n"); complete: ssl_stream_free(ctx->s_stream); evutil_closesocket(fd); ssl_shutdown_ctx_free(ctx); } /* * Cleanly shutdown an SSL session on file descriptor fd using low-level * file descriptor readiness events on event base evbase. * Guarantees that SSL and the corresponding SSL_CTX are freed and the * socket is closed, eventually, or in the case of fatal errors, immediately. */ void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, evutil_socket_t fd) { struct ssl_shutdown_ctx * sslshutctx = NULL; sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); pxy_ssl_shutdown_cb(fd, 0, sslshutctx); return; }