#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SSL_EX_DATA_IDX_SSLMGR 0 struct ssl_mgr { uint8_t sslcomp; uint8_t no_ssl2; uint8_t no_ssl3; uint8_t no_tls10; uint8_t no_tls11; uint8_t no_tls12; CONST_SSL_METHOD *(*sslmethod)(void); int sslversion; char ssl_session_context[8]; MESA_htable_handle down_sess_cache; MESA_htable_handle up_sess_cache; char* default_ciphers; DH * dh; char * ecdhcurve; char * crlurl; uint8_t SSL_MODE_RELEASE_BUFFERS; void *logger; }; struct ssl_stream { enum tfe_conn_dir dir; SSL * ssl; struct ssl_mgr* ref_mgr; char* sni; }; struct peek_client_hello_ctx { unsigned char sni_peek_retries; /* max 64 SNI parse retries */ struct event* ev; struct event_base* evbase; }; struct ssl_connect_origin_ctx { struct bufferevent *bev; struct ssl_stream* ssl; struct sockaddr addr; socklen_t addrlen; }; /* * SSL shutdown context. */ struct ssl_shutdown_ctx { struct ssl_stream* s_stream; struct event_base *evbase; struct event *ev; unsigned int retries; }; struct ssl_mgr* init_ssl_manager(const char* ini_profile, const char* section, void* logger) { struct ssl_mgr* mgr=ALLOC(struct ssl_mgr, 1); mgr->logger=logger; memcpy(mgr->ssl_session_context,"mesa-tfe",sizeof(mgr->ssl_session_context)); return mgr; } void destroy_ssl_manager(struct ssl_mgr* mgr) { free(mgr); } SSL_SESSION* up_session_get(MESA_htable_handle* cache, struct sockaddr * addr, socklen_t addr_len, const char* sni) { } void up_session_set(MESA_htable_handle* cache, struct sockaddr * addr, socklen_t addr_len, const char* sni, SSL_SESSION * value) { } SSL_SESSION* down_session_get(MESA_htable_handle* cache, unsigned char * id, int idlen) { dynbuf_t *valbuf = (dynbuf_t *)val; MESA_htable_search_cb(const MESA_htable_handle table, const uchar * key, uint size, hash_cb_fun_t * cb, void * arg, long * cb_ret); SSL_SESSION *sess; const unsigned char *p; p = (const unsigned char *)valbuf->buf; sess = d2i_SSL_SESSION(NULL, &p, valbuf->sz); /* increments p */ if (!sess) return NULL; if (!ssl_session_is_valid(sess)) { SSL_SESSION_free(sess); return NULL; } if (copy) return sess; SSL_SESSION_free(sess); return ((void*)-1); } void down_session_set(MESA_htable_handle* cache, SSL_SESSION* sess) { unsigned int len=0; const unsigned char* id = SSL_SESSION_get_id(sess, &len); unsigned char *p=NULL; size_t asn1sz=0; asn1sz = i2d_SSL_SESSION(sess, NULL); p=ALLOC(char, asn1sz); i2d_SSL_SESSION(sess, &p); /* updates p */ int ret=MESA_htable_add(cache, id, len, p); return; } void down_session_del(MESA_htable_handle* cache, SSL_SESSION* sess) { } void peek_client_hello_ctx_free(void* ctx) { struct peek_client_hello_ctx * _ctx=(struct peek_client_hello_ctx *)ctx; event_free(_ctx->ev); _ctx->ev = NULL; free(_ctx->sni); _ctx->sni=NULL; free(_ctx); return; } struct ssl_chello* ssl_peek_result_release_chello(future_result_t* result) { struct ssl_chello* p=(struct ssl_chello* )result, *copy=NULL; copy=ALLOC(struct ssl_chello*,1); if(p!=NULL) { copy->sni=tfe_strdup(p->sni); copy->cipher_suites=tfe_strdup(p->cipher_suites); copy->version=p->version; } return copy; } void ssl_free_chello(struct ssl_chello* p) { if(p==NULL) { return; } free(p->sni); p->sni=NULL; free(p->cipher_suites); p->cipher_suites=NULL; p->cipher_suites=NULL; free(p); return; } static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) { struct promise* promise=(struct promise*)arg; struct peek_client_hello_ctx* ctx= (struct peek_client_hello_ctx*)promise_get_ctx(promise); struct ssl_chello *result=NULL; const char* reason_too_many_retries="too many tries"; const char* reason_see_no_client_hello="see no client hello"; const char* reason=NULL; memset(&result, 0, sizeof(result)); char* sni=NULL; unsigned char buf[1024]; ssize_t n=0; const unsigned char * chello=NULL; int rv=0; n = recv(fd, buf, sizeof(buf), MSG_PEEK); if (n == -1) { TFE_LOG_ERROR("Error peeking on fd, aborting connection\n"); goto failed; } if (n == 0) { goto failed; } result=ssl_get_peek_result(NULL); //todo: parse version and cipher suites. //or we should use sni proxy instead? https://github.com/dlundquist/sniproxy/blob/master/src/tls.c rv = ssl_tls_clienthello_parse(buf, n, 0, &chello, &result->sni); if(rv==0) { promise_dettach_ctx(promise); promise_success(promise, result); ssl_free_chello(result); peek_client_hello_ctx_free(ctx); } else { if(!chello) { TFE_LOG_ERROR("Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); reason="see no client hello"; goto failed; } if(ctx->sni_peek_retries++ > 50) { TFE_LOG_ERROR("Peek failed due to too many retries\n"); reason="too many peek retries"; goto failed; } /* ssl_tls_clienthello_parse indicates that we * should retry later when we have more data, and we * haven't reached the maximum retry count yet. * Reschedule this event as timeout-only event in * order to prevent busy looping over the read event. * Because we only peeked at the pending bytes and * never actually read them, fd is still ready for * reading now. We use 25 * 0.2 s = 5 s timeout. */ struct timeval retry_delay = {0, 100}; event_free(ctx->ev); ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); assert(ctx->ev!=NULL); event_add(ctx->ev, &retry_delay); } return; failed: promise_dettach_ctx(promise); promise_failed(promise,FUTURE_ERROR_EXCEPTION,reason); peek_client_hello_ctx_free(ctx); ssl_free_chello(result); return; } void ssl_async_peek_client_hello(struct future* future, evutil_socket_t fd, struct event_base *evbase) { struct event * ev=NULL; struct promise* p=future_to_promise(future); struct peek_client_hello_ctx* ctx=ALLOC(struct peek_client_hello_ctx, 1); ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); event_add(evbase, NULL); promise_set_ctx(p, ctx, peek_client_hello_ctx_free); return; } /* * Create new SSL context for outgoing connections to the original destination. * If hostname sni is provided, use it for Server Name Indication. */ static SSL* upstream_ssl_create(struct ssl_mgr* mgr, const struct ssl_chello* chello, struct sockaddr* addr, socklen_t addrlen) { SSL_CTX* sslctx=NULL; SSL* ssl=NULL; SSL_SESSION* sess=NULL; sslctx = SSL_CTX_new(mgr->sslmethod()); sslctx_set_opts(sslctx, mgr); #if OPENSSL_VERSION_NUMBER >= 0x10100000L if (mgr->sslversion) { if (SSL_CTX_set_min_proto_version(sslctx, chello->version) == 0 || SSL_CTX_set_max_proto_version(sslctx, chello->version) == 0) { SSL_CTX_free(sslctx); return NULL; } } #endif /* OPENSSL_VERSION_NUMBER >= 0x10100000L */ SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); ssl = SSL_new(sslctx); SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ if (!ssl) { return NULL; } #ifndef OPENSSL_NO_TLSEXT if (chello->sni) { SSL_set_tlsext_host_name(ssl, sni); } #endif /* !OPENSSL_NO_TLSEXT */ #ifdef SSL_MODE_RELEASE_BUFFERS /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); #endif /* SSL_MODE_RELEASE_BUFFERS */ //Todo: Refactor below session resumption. /* session resuming based on remote endpoint address and port */ sess = up_session_get(mgr->up_sess_cache, addr, addrlen, chello->sni); /* new sess inst */ if (sess) { SSL_set_session(ssl, sess); /* increments sess refcount */ SSL_SESSION_free(sess); } return ssl; } void ssl_stream_shutdown(struct bufferevent *bev,struct ssl_stream* ssl) { //TODO: } void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx* ctx) { if(ctx->ssl!=NULL) { ssl_stream_shutdown(ctx->bev,ctx->ssl); free(ctx->ssl); } if(ctx->bev!=NULL) { bufferevent_free(ctx->bev); ctx->bev=NULL; } free(ctx); return; } struct ssl_stream* ssl_conn_origin_result_release_stream(future_result_t* result) { struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx* )result; struct ssl_stream* ret=ctx->ssl; ctx->ssl=NULL; //giveup ownership return ret; } struct bufferevent* ssl_conn_origin_result_release_bev(future_result_t* result) { struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx* )result; struct bufferevent * ret=ctx->bev; ctx->bev=NULL; //giveup ownership return ret; } /* * Callback for meta events on the up- and downstream connection bufferevents. * Called when EOF has been reached, a connection has been made, and on errors. */ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg) { struct promise* promise=(struct promise*)arg; struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx*)promise_get_ctx(promise); SSL_SESSION * to_cached=NULL; struct tfe_stream_private* _stream = (struct tfe_stream_private*)arg; if (events & BEV_EVENT_ERROR) { promise_failed(promise, FUTURE_ERROR_EXCEPTION, "ssl connect failed"); } else { if (events & BEV_EVENT_CONNECTED) { bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL);//leave a clean bev for on_success to_cached=SSL_get0_session(ctx->ssl->ssl); up_session_set(ctx->ssl->ref_mgr->up_sess_cache, &(ctx->addr), ctx->addrlen, ctx->ssl->sni, to_cached); promise_dettach_ctx(promise); promise_success(promise, ctx); ctx->bev=NULL; } else { assert(0); } } ssl_connect_origin_ctx_free(ctx); return } void ssl_async_connect_origin(struct future* future, struct ssl_mgr* mgr, const struct ssl_chello* chello, evutil_socket_t fd, struct event_base *evbase) { struct promise* p=future_to_promise(future); struct ssl_connect_origin_ctx* ctx=ALLOC(struct ssl_connect_origin_ctx, 1); int ret=0; struct sockaddr addr; socklen_t addrlen; ret=getpeername(fd, &(ctx->addr), &(ctx->addrlen)); assert(ret==0); ctx->ssl=ALLOC(struct ssl_stream,1); ctx->ssl->ref_mgr=mgr; ctx->ssl->dir=CONN_DIR_UPSTREAM; ctx->ssl->sni=tfe_strdup(chello->sni); ctx->ssl->ssl=upstream_ssl_create(mgr, chello, &(ctx->addr), ctx->addrlen); ctx->bev = bufferevent_openssl_socket_new(evbase, fd, ctx->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1); bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, p); bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); } /* * Called by OpenSSL when a new src SSL session is created. * OpenSSL increments the refcount before calling the callback and will * decrement it again if we return 0. Returning 1 will make OpenSSL skip * the refcount decrementing. In other words, return 0 if we did not * keep a pointer to the object (which we never do here). */ static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) { struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); #ifdef HAVE_SSLV2 /* Session resumption seems to fail for SSLv2 with protocol * parsing errors, so we disable caching for SSLv2. */ if (SSL_version(ssl) == SSL2_VERSION) { return 0; } #endif /* HAVE_SSLV2 */ if (sess) { down_session_set(mgr->down_sess_cache, sess); } return 0; } /* * Called by OpenSSL when a src SSL session should be removed. * OpenSSL calls SSL_SESSION_free() after calling the callback; * we do not need to free the reference here. */ static void ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) { struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); if (sess) { down_session_del(mgr->down_sess_cache, sess); } return; } /* * Called by OpenSSL when a src SSL session is requested by the client. OPENSSL_VERSION_NUMBER >= 0x10100000L required. */ static SSL_SESSION * ossl_sessget_cb(UNUSED SSL *ssl, const unsigned char *id, int idlen, int *copy) { struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); SSL_SESSION * sess; *copy = 0; /* SSL should not increment reference count of session */ sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache,id, idlen); return sess; } /* * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX. */ static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr* mgr) { SSL_CTX_set_options(sslctx, SSL_OP_ALL); #ifdef SSL_OP_TLS_ROLLBACK_BUG SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG); #endif /* SSL_OP_TLS_ROLLBACK_BUG */ #ifdef SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); #endif /* SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION */ #ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); #endif /* SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS */ #ifdef SSL_OP_NO_TICKET SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET); #endif /* SSL_OP_NO_TICKET */ #ifdef SSL_OP_NO_SSLv2 #ifdef HAVE_SSLV2 if (mgr->no_ssl2) { #endif /* HAVE_SSLV2 */ SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); #ifdef HAVE_SSLV2 } #endif /* HAVE_SSLV2 */ #endif /* !SSL_OP_NO_SSLv2 */ #ifdef HAVE_SSLV3 if (mgr->no_ssl3) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); } #endif /* HAVE_SSLV3 */ #ifdef HAVE_TLSV10 if (mgr->no_tls10) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1); } #endif /* HAVE_TLSV10 */ #ifdef HAVE_TLSV11 if (mgr->no_tls11) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1); } #endif /* HAVE_TLSV11 */ #ifdef HAVE_TLSV12 if (mgr->no_tls12) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2); } #endif /* HAVE_TLSV12 */ #ifdef SSL_OP_NO_COMPRESSION if (!mgr->sslcomp) { SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION); } #endif /* SSL_OP_NO_COMPRESSION */ SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); } /* * Create and set up a new SSL_CTX instance for terminating SSL. * Set up all the necessary callbacks, the cert, the cert chain and key. */ static SSL_CTX * down_sslctx_create(struct ssl_mgr* mgr, struct cert* crt) { SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); if (!sslctx) return NULL; sslctx_set_opts(sslctx, mgr); //TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L if (mgr->sslversion) { if (SSL_CTX_set_min_proto_version(sslctx,mgr->sslversion) == 0 || SSL_CTX_set_max_proto_version(sslctx, mgr->sslversion) == 0) { SSL_CTX_free(sslctx); return NULL; } } SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); #ifdef USE_SSL_SESSION_ID_CONTEXT SSL_CTX_set_session_id_context(sslctx, (void *)(mgr->ssl_session_context), sizeof(mgr->ssl_session_context)); #endif /* USE_SSL_SESSION_ID_CONTEXT */ #ifndef OPENSSL_NO_DH if (mgr->dh) { SSL_CTX_set_tmp_dh(sslctx, mgr->dh); } else { SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); } #endif /* !OPENSSL_NO_DH */ #ifndef OPENSSL_NO_ECDH if (mgr->ecdhcurve) { EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } else { EC_KEY * ecdh = ssl_ec_by_name(NULL); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } #endif /* !OPENSSL_NO_ECDH */ SSL_CTX_use_certificate(sslctx, crt->crt); SSL_CTX_use_PrivateKey(sslctx, crt->key); for (int i = 0; i < sk_X509_num(crt->chain); i++) { X509 * c = sk_X509_value(crt->chain, i); ssl_x509_refcount_inc(c); /* next call consumes a reference */ SSL_CTX_add_extra_chain_cert(sslctx, c); } return sslctx; } /* * Create new SSL context for the incoming connection, based on the a designate cert * Returns NULL when openssl error. */ struct ssl_stream * ssl_downstream_create(struct ssl_mgr* mgr, struct ssl_chello* hello, struct cert* crt) { struct ssl_stream* p=NULL; if(crt==NULL) { goto error_out; } p=ALLOC(struct ssl_stream, 1); p->ref_mgr=mgr; p->dir=CONN_DIR_DOWNSTREAM; //could be optimized by store the sslctx. SSL_CTX * sslctx = down_sslctx_create(mgr,crt); cert_mgr_free_cert(crt); if (!sslctx) { goto error_out; } p->ssl = SSL_new(sslctx); int ret=SSL_set_ex_data(p->ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); assert(ret==1); SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ if (!p->ssl) { goto error_out; } if(mgr->SSL_MODE_RELEASE_BUFFERS==1) { /* lower memory footprint for idle connections */ SSL_set_mode( p->ssl, SSL_get_mode(p->ssl) | SSL_MODE_RELEASE_BUFFERS); } p->sni=tfe_strdup(hello->sni); return p; error_out: free(p); return NULL; } /* * Cleanly shut down an SSL socket. Libevent currently has no support for * cleanly shutting down an SSL socket so we work around that by using a * low-level event. This works for recent versions of OpenSSL. OpenSSL * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE * may or may not work. */ static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream* s_stream, struct event_base *evbase) { struct ssl_shutdown_ctx *ctx = ALLOC(struct ssl_shutdown_ctx,1); ctx->evbase = evbase; ctx->s_stream = s_stream; ctx->ev = NULL; ctx->retries = 0; return ctx; } static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx *ctx) { free(ctx); } static void ssl_stream_free(struct ssl_stream* s_stream) { SSL_free(s_stream->ssl); s_stream->ssl=NULL; free(s_stream->sni); s_stream->sni=NULL; s_stream->ref_mgr=NULL free(s_stream); return; } /* * The shutdown socket event handler. This is either * scheduled as a timeout-only event, or as a fd read or * fd write event, depending on whether SSL_shutdown() * indicates it needs read or write on the socket. */ static void pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void *arg) { struct ssl_shutdown_ctx *ctx = (struct ssl_shutdown_ctx *)arg; struct timeval retry_delay = {0, 100}; void *logger=ctx->s_stream->ref_mgr->logger; short want = 0; int rv=0, sslerr=0; if (ctx->ev) { event_free(ctx->ev); ctx->ev = NULL; } /* * Use the new (post-2008) semantics for SSL_shutdown() on a * non-blocking socket. SSL_shutdown() returns -1 and WANT_READ * if the other end's close notify was not received yet, and * WANT_WRITE it could not write our own close notify. * * This is a good collection of recent and relevant documents: * http://bugs.python.org/issue8108 */ rv = SSL_shutdown(ctx->s_stream->ssl); if (rv == 1) goto complete; if (rv != -1) { goto retry; } switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) { case SSL_ERROR_WANT_READ: want = EV_READ; goto retry; case SSL_ERROR_WANT_WRITE: want = EV_WRITE; goto retry; case SSL_ERROR_ZERO_RETURN: goto retry; case SSL_ERROR_SYSCALL: case SSL_ERROR_SSL: goto complete; default: TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() " "error %i. Closing fd.\n", sslerr); goto complete; } goto complete; retry: if (ctx->retries++ >= 50) { TFE_LOG_ERROR(logger,"Failed to shutdown SSL connection cleanly: " "Max retries reached. Closing fd.\n"); goto complete; } ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); if (ctx->ev) { event_add(ctx->ev, &retry_delay); return; } TFE_LOG_ERROR(logger,"Failed to shutdown SSL connection cleanly: " "Cannot create event. Closing fd.\n"); complete: ssl_stream_free(ctx->s_stream); evutil_closesocket(fd); ssl_shutdown_ctx_free(ctx); } /* * Cleanly shutdown an SSL session on file descriptor fd using low-level * file descriptor readiness events on event base evbase. * Guarantees that SSL and the corresponding SSL_CTX are freed and the * socket is closed, eventually, or in the case of fatal errors, immediately. */ void ssl_stream_free_and_close_fd(struct ssl_stream* s_stream, struct event_base *evbase, evutil_socket_t fd) { struct ssl_shutdown_ctx *sslshutctx=NULL; sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); pxy_ssl_shutdown_cb(fd, 0, sslshutctx); return; }