diff --git a/platform/include/internal/ssl_stream.h b/platform/include/internal/ssl_stream.h index 806822c..47b619f 100644 --- a/platform/include/internal/ssl_stream.h +++ b/platform/include/internal/ssl_stream.h @@ -8,7 +8,7 @@ struct ssl_stream; struct ssl_mgr; -struct ssl_mgr* init_ssl_manager(const char* ini_profile, const char* section); +struct ssl_mgr* init_ssl_manager(const char* ini_profile, const char* section, void* logger); void destroy_ssl_manager(struct ssl_mgr* mgr); struct ssl_chello diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index d29ea7f..ba0c5de 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -48,9 +48,11 @@ struct ssl_mgr char * ecdhcurve; char * crlurl; uint8_t SSL_MODE_RELEASE_BUFFERS; + void *logger; }; struct ssl_stream { + enum tfe_conn_dir dir; SSL * ssl; struct ssl_mgr* ref_mgr; char* sni; @@ -70,10 +72,20 @@ struct ssl_connect_origin_ctx struct sockaddr addr; socklen_t addrlen; }; +/* + * SSL shutdown context. + */ +struct ssl_shutdown_ctx { + struct ssl_stream* s_stream; + struct event_base *evbase; + struct event *ev; + unsigned int retries; +}; -struct ssl_mgr* init_ssl_manager(const char* ini_profile, const char* section) +struct ssl_mgr* init_ssl_manager(const char* ini_profile, const char* section, void* logger) { - struct ssl_mgr* mgr=ALLOC(struct ssl_mgr*, 1); + struct ssl_mgr* mgr=ALLOC(struct ssl_mgr, 1); + mgr->logger=logger; memcpy(mgr->ssl_session_context,"mesa-tfe",sizeof(mgr->ssl_session_context)); return mgr; } @@ -267,7 +279,7 @@ static SSL* upstream_ssl_create(struct ssl_mgr* mgr, const struct ssl_chello* ch sslctx = SSL_CTX_new(mgr->sslmethod()); - pxy_sslctx_setoptions(sslctx, mgr); + sslctx_set_opts(sslctx, mgr); #if OPENSSL_VERSION_NUMBER >= 0x10100000L if (mgr->sslversion) { @@ -394,6 +406,7 @@ void ssl_async_connect_origin(struct future* future, struct ssl_mgr* mgr, const assert(ret==0); ctx->ssl=ALLOC(struct ssl_stream,1); ctx->ssl->ref_mgr=mgr; + ctx->ssl->dir=CONN_DIR_UPSTREAM; ctx->ssl->sni=tfe_strdup(chello->sni); ctx->ssl->ssl=upstream_ssl_create(mgr, chello, &(ctx->addr), ctx->addrlen); ctx->bev = bufferevent_openssl_socket_new(evbase, fd, ctx->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); @@ -403,194 +416,6 @@ void ssl_async_connect_origin(struct future* future, struct ssl_mgr* mgr, const bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); } -/* - * Dump information on a cert to the debug log. - */ -static void pxy_debug_crt(X509 * crt) -{ - char * sj = ssl_x509_subject(crt); - if (sj) - { - log_dbg_printf("Subject DN: %s\n", sj); - free(sj); - } - - char * names = ssl_x509_names_to_str(crt); - if (names) - { - log_dbg_printf("Common Names: %s\n", names); - free(names); - } - - char * fpr; - if (!(fpr = ssl_x509_fingerprint(crt, 1))) - { - log_err_printf("Warning: Error generating X509 fingerprint\n"); - } - else - { - log_dbg_printf("Fingerprint: %s\n", fpr); - free(fpr); - } - -#ifdef DEBUG_CERTIFICATE - /* dump cert */ - log_dbg_print_free(ssl_x509_to_str(crt)); - log_dbg_print_free(ssl_x509_to_pem(crt)); -#endif /* DEBUG_CERTIFICATE */ -} - -static void -pxy_log_connect_nonhttp(pxy_conn_ctx_t * ctx) -{ - char * msg; - int rv; - - /* - * The following ifdef's within asprintf arguments list generates - * warnings with -Wembedded-directive on some compilers. - * Not fixing the code in order to avoid more code duplication. - */ - - if (!ctx->src.ssl) - { - rv = asprintf(&msg, "%s %s %s %s %s" - #ifdef HAVE_LOCAL_PROCINFO - " %s" - #endif /* HAVE_LOCAL_PROCINFO */ - "\n", - ctx->passthrough ? "passthrough" : "tcp", - STRORDASH(ctx->srchost_str), - STRORDASH(ctx->srcport_str), - STRORDASH(ctx->dsthost_str), - STRORDASH(ctx->dstport_str) - ); - } - else - { - rv = asprintf(&msg, "%s %s %s %s %s " - "sni:%s names:%s " - "sproto:%s:%s dproto:%s:%s " - "origcrt:%s usedcrt:%s\n", - ctx->clienthello_found ? "upgrade" : "ssl", - STRORDASH(ctx->srchost_str), - STRORDASH(ctx->srcport_str), - STRORDASH(ctx->dsthost_str), - STRORDASH(ctx->dstport_str), - STRORDASH(ctx->sni), - STRORDASH(ctx->ssl_names), - SSL_get_version(ctx->src.ssl), - SSL_get_cipher(ctx->src.ssl), - SSL_get_version(ctx->dst.ssl), - SSL_get_cipher(ctx->dst.ssl), - STRORDASH(ctx->origcrtfpr), - STRORDASH(ctx->usedcrtfpr) - ); - } - if ((rv < 0) || !msg) - { - ctx->enomem = 1; - goto out; - } - if (!ctx->opts->detach) - { - log_err_printf("%s", msg); - } - if (ctx->opts->connectlog) - { - } - else - { - free(msg); - } -out: - return; -} - -static void -pxy_log_connect_http(pxy_conn_ctx_t * ctx) -{ - char * msg; - int rv; - -#ifdef DEBUG_PROXY - if (ctx->passthrough) { - log_err_printf("Warning: pxy_log_connect_http called while in " - "passthrough mode\n"); - return; - } -#endif - - - /* - * The following ifdef's within asprintf arguments list generates - * warnings with -Wembedded-directive on some compilers. - * Not fixing the code in order to avoid more code duplication. - */ - - if (!ctx->spec->ssl) - { - rv = asprintf(&msg, "http %s %s %s %s %s %s %s %s %s %s\n", - STRORDASH(ctx->srchost_str), - STRORDASH(ctx->srcport_str), - STRORDASH(ctx->dsthost_str), - STRORDASH(ctx->dstport_str), - STRORDASH(ctx->http_host), - STRORDASH(ctx->http_method), - STRORDASH(ctx->http_uri), - STRORDASH(ctx->http_status_code), - STRORDASH(ctx->http_content_length), - ctx->ocsp_denied ? " ocsp:denied" : ""); - } - else - { - rv = asprintf(&msg, "https %s %s %s %s %s %s %s %s %s " - "sni:%s names:%s " - "sproto:%s:%s dproto:%s:%s " - "origcrt:%s usedcrt:%s" - #ifdef HAVE_LOCAL_PROCINFO - " %s" - #endif /* HAVE_LOCAL_PROCINFO */ - "%s\n", - STRORDASH(ctx->srchost_str), - STRORDASH(ctx->srcport_str), - STRORDASH(ctx->dsthost_str), - STRORDASH(ctx->dstport_str), - STRORDASH(ctx->http_host), - STRORDASH(ctx->http_method), - STRORDASH(ctx->http_uri), - STRORDASH(ctx->http_status_code), - STRORDASH(ctx->http_content_length), - STRORDASH(ctx->sni), - STRORDASH(ctx->ssl_names), - SSL_get_version(ctx->src.ssl), - SSL_get_cipher(ctx->src.ssl), - SSL_get_version(ctx->dst.ssl), - SSL_get_cipher(ctx->dst.ssl), - STRORDASH(ctx->origcrtfpr), - STRORDASH(ctx->usedcrtfpr), - ctx->ocsp_denied ? " ocsp:denied" : ""); - } - if ((rv < 0) || !msg) - { - ctx->enomem = 1; - goto out; - } - if (!ctx->opts->detach) - { - log_err_printf("%s", msg); - } - if (ctx->opts->connectlog) - { - } - - else - { - free(msg); - } -out: - return; -} /* * Called by OpenSSL when a new src SSL session is created. @@ -601,7 +426,7 @@ out: */ static int -pxy_ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) +ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) { struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); @@ -625,7 +450,7 @@ pxy_ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) * we do not need to free the reference here. */ static void -pxy_ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) +ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) { struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); if (sess) @@ -640,7 +465,7 @@ pxy_ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) OPENSSL_VERSION_NUMBER >= 0x10100000L required. */ static SSL_SESSION * -pxy_ossl_sessget_cb(UNUSED SSL *ssl, const unsigned char *id, int idlen, int *copy) +ossl_sessget_cb(UNUSED SSL *ssl, const unsigned char *id, int idlen, int *copy) { struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); SSL_SESSION * sess; @@ -654,7 +479,7 @@ pxy_ossl_sessget_cb(UNUSED SSL *ssl, const unsigned char *id, int idlen, int *co * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX. */ static void -pxy_sslctx_setoptions(SSL_CTX * sslctx, struct ssl_mgr* mgr) +sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr* mgr) { SSL_CTX_set_options(sslctx, SSL_OP_ALL); #ifdef SSL_OP_TLS_ROLLBACK_BUG @@ -715,223 +540,6 @@ pxy_sslctx_setoptions(SSL_CTX * sslctx, struct ssl_mgr* mgr) } -static int -pxy_srccert_write_to_gendir(pxy_conn_ctx_t * ctx, X509 * crt, int is_orig) -{ - char * fn; - int rv; - - if (!ctx->origcrtfpr) - return -1; - if (is_orig) - { - rv = asprintf(&fn, "%s/%s.crt", ctx->opts->certgendir, - ctx->origcrtfpr); - } - else - { - if (!ctx->usedcrtfpr) - return -1; - rv = asprintf(&fn, "%s/%s-%s.crt", ctx->opts->certgendir, - ctx->origcrtfpr, ctx->usedcrtfpr); - } - if (rv == -1) - { - ctx->enomem = 1; - return -1; - } - - free(fn); - return rv; -} - -static void pxy_srccert_write(pxy_conn_ctx_t * ctx) -{ - if (ctx->opts->certgen_writeall || ctx->generated_cert) - { - if (pxy_srccert_write_to_gendir(ctx, - SSL_get_certificate(ctx->src.ssl), 0) == -1) - { - log_err_printf("Failed to write used cert\n"); - } - } - if (ctx->opts->certgen_writeall) - { - if (pxy_srccert_write_to_gendir(ctx, ctx->origcrt, 1) == -1) - { - log_err_printf("Failed to write orig cert\n"); - } - } -} - -/* - * Create new SSL context for the incoming connection, based on the original - * destination SSL cert. - * Returns NULL if no suitable cert could be found. - */ - - -static SSL * downstream_ssl_create(cert_t * cert, struct tfe_config* opts) -{ - - assert(cert!=NULL); - SSL_CTX * sslctx = downstream_sslctx_create(ctx, cert->crt, cert->chain, - cert->key); - - if (!sslctx) - { - return NULL; - } - SSL * ssl = SSL_new(sslctx); - SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ - if (!ssl) - { - return NULL; - } -#ifdef SSL_MODE_RELEASE_BUFFERS - /* lower memory footprint for idle connections */ - SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); -#endif /* SSL_MODE_RELEASE_BUFFERS */ - return ssl; -} - -/* - * OpenSSL servername callback, called when OpenSSL receives a servername - * TLS extension in the clientHello. Must switch to a new SSL_CTX with - * a different cert if we want to replace the server cert here. - * We generate a new cert if the current one does not match the - * supplied servername. This should only happen if the original destination - * server supplies a cert which does not match the server name we - * indicate to it. - */ -static int pxy_ossl_servername_cb(SSL * ssl, UNUSED int * al, void * arg) -{ - pxy_conn_ctx_t * ctx = (pxy_conn_ctx_t *) arg; - const char * sn; - X509 * sslcrt; - - if (!(sn = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name))) - return SSL_TLSEXT_ERR_NOACK; - - if (!ctx->sni) - { - if (OPTS_DEBUG(ctx->opts)) - { - log_dbg_printf("Warning: SNI parser yielded no " - "hostname, copying OpenSSL one: " - "[NULL] != [%s]\n", sn); - } - ctx->sni = strdup(sn); - if (!ctx->sni) - { - ctx->enomem = 1; - return SSL_TLSEXT_ERR_NOACK; - } - } - if (OPTS_DEBUG(ctx->opts)) - { - if (strcmp(sn, ctx->sni) != 0) - { - /* - * This may happen if the client resumes a session, but - * uses a different SNI hostname when resuming than it - * used when the session was created. OpenSSL - * correctly ignores the SNI in the ClientHello in this - * case, but since we have already sent the SNI onwards - * to the original destination, there is no way back. - * We log an error and hope this never happens. - */ - log_dbg_printf("Warning: SNI parser yielded different " - "hostname than OpenSSL callback for " - "the same ClientHello message: " - "[%s] != [%s]\n", ctx->sni, sn); - } - } - - /* generate a new cert with sn as additional altSubjectName - * and replace it both in the current SSL ctx and in the cert cache */ - if (!ctx->immutable_cert && - !ssl_x509_names_match((sslcrt = SSL_get_certificate(ssl)), sn)) - { - X509 * newcrt; - SSL_CTX * newsslctx; - - if (OPTS_DEBUG(ctx->opts)) - { - log_dbg_printf("Certificate cache: UPDATE " - "(SNI mismatch)\n"); - } - newcrt = ssl_x509_forge(ctx->opts->cacrt, ctx->opts->cakey, - sslcrt, ctx->opts->key, - sn, ctx->opts->crlurl); - if (!newcrt) - { - ctx->enomem = 1; - return SSL_TLSEXT_ERR_NOACK; - } - cachemgr_fkcrt_set(ctx->origcrt, newcrt); - ctx->generated_cert = 1; - if (OPTS_DEBUG(ctx->opts)) - { - log_dbg_printf("===> Updated forged server " - "cert:\n"); - pxy_debug_crt(newcrt); - } - if (WANT_CONNECT_LOG(ctx)) - { - if (ctx->ssl_names) - { - free(ctx->ssl_names); - } - ctx->ssl_names = ssl_x509_names_to_str(newcrt); - if (!ctx->ssl_names) - { - ctx->enomem = 1; - } - } - if (WANT_CONNECT_LOG(ctx) || ctx->opts->certgendir) - { - if (ctx->usedcrtfpr) - { - free(ctx->usedcrtfpr); - } - ctx->usedcrtfpr = ssl_x509_fingerprint(newcrt, 0); - if (!ctx->usedcrtfpr) - { - ctx->enomem = 1; - } - } - - newsslctx = downstream_sslctx_create(ctx, newcrt, ctx->opts->chain, - ctx->opts->key); - if (!newsslctx) - { - X509_free(newcrt); - ctx->enomem = 1; - return SSL_TLSEXT_ERR_NOACK; - } - SSL_set_SSL_CTX(ssl, newsslctx); /* decr's old incr new refc */ - SSL_CTX_free(newsslctx); - X509_free(newcrt); - } - else if (OPTS_DEBUG(ctx->opts)) - { - log_dbg_printf("Certificate cache: KEEP (SNI match or " - "target mode)\n"); - } - - return SSL_TLSEXT_ERR_OK; -} - - - -void ssl_upstream_free(struct ssl_upstream* p) -{ - X509_free(p->orig_cert); - //todo close ssl with a callback. - //SSL_free(ctx->ssl); -} - /* * Create and set up a new SSL_CTX instance for terminating SSL. * Set up all the necessary callbacks, the cert, the cert chain and key. @@ -942,7 +550,7 @@ static SSL_CTX * down_sslctx_create(struct ssl_mgr* mgr, struct cert* crt) if (!sslctx) return NULL; - pxy_sslctx_setoptions(sslctx, mgr); + sslctx_set_opts(sslctx, mgr); //TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L if (mgr->sslversion) { @@ -953,9 +561,9 @@ static SSL_CTX * down_sslctx_create(struct ssl_mgr* mgr, struct cert* crt) } } - SSL_CTX_sess_set_new_cb(sslctx, pxy_ossl_sessnew_cb); - SSL_CTX_sess_set_remove_cb(sslctx, pxy_ossl_sessremove_cb); - SSL_CTX_sess_set_get_cb(sslctx, pxy_ossl_sessget_cb); + SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); + SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); + SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); #ifdef USE_SSL_SESSION_ID_CONTEXT @@ -1000,9 +608,8 @@ static SSL_CTX * down_sslctx_create(struct ssl_mgr* mgr, struct cert* crt) /* - * Create new SSL context for the incoming connection, based on the original - * destination SSL cert. - * Returns NULL if no suitable cert could be found. + * Create new SSL context for the incoming connection, based on the a designate cert + * Returns NULL when openssl error. */ struct ssl_stream * ssl_downstream_create(struct ssl_mgr* mgr, struct ssl_chello* hello, struct cert* crt) { @@ -1013,6 +620,7 @@ struct ssl_stream * ssl_downstream_create(struct ssl_mgr* mgr, struct ssl_chello } p=ALLOC(struct ssl_stream, 1); p->ref_mgr=mgr; + p->dir=CONN_DIR_DOWNSTREAM; //could be optimized by store the sslctx. SSL_CTX * sslctx = down_sslctx_create(mgr,crt); cert_mgr_free_cert(crt); @@ -1040,11 +648,125 @@ error_out: return NULL; } -void ssl_stream_free_and_close_fd(struct ssl_stream* stream, struct event_base *evbase, evutil_socket_t fd) + +/* + * Cleanly shut down an SSL socket. Libevent currently has no support for + * cleanly shutting down an SSL socket so we work around that by using a + * low-level event. This works for recent versions of OpenSSL. OpenSSL + * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE + * may or may not work. + */ +static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream* s_stream, struct event_base *evbase) { - free(stream->sni); - stream->sni=NULL; -// X509_free(p->fake_cert); + struct ssl_shutdown_ctx *ctx = ALLOC(struct ssl_shutdown_ctx,1); + ctx->evbase = evbase; + ctx->s_stream = s_stream; + ctx->ev = NULL; + ctx->retries = 0; + return ctx; +} + +static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx *ctx) +{ + free(ctx); +} +static void ssl_stream_free(struct ssl_stream* s_stream) +{ + SSL_free(s_stream->ssl); + s_stream->ssl=NULL; + free(s_stream->sni); + s_stream->sni=NULL; + s_stream->ref_mgr=NULL + free(s_stream); + return; +} +/* + * The shutdown socket event handler. This is either + * scheduled as a timeout-only event, or as a fd read or + * fd write event, depending on whether SSL_shutdown() + * indicates it needs read or write on the socket. + */ +static void +pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void *arg) +{ + struct ssl_shutdown_ctx *ctx = (struct ssl_shutdown_ctx *)arg; + struct timeval retry_delay = {0, 100}; + void *logger=ctx->s_stream->ref_mgr->logger; + short want = 0; + int rv=0, sslerr=0; + + if (ctx->ev) { + event_free(ctx->ev); + ctx->ev = NULL; + } + + /* + * Use the new (post-2008) semantics for SSL_shutdown() on a + * non-blocking socket. SSL_shutdown() returns -1 and WANT_READ + * if the other end's close notify was not received yet, and + * WANT_WRITE it could not write our own close notify. + * + * This is a good collection of recent and relevant documents: + * http://bugs.python.org/issue8108 + */ + rv = SSL_shutdown(ctx->s_stream->ssl); + if (rv == 1) + goto complete; + if (rv != -1) { + goto retry; + } + switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) { + case SSL_ERROR_WANT_READ: + want = EV_READ; + goto retry; + case SSL_ERROR_WANT_WRITE: + want = EV_WRITE; + goto retry; + case SSL_ERROR_ZERO_RETURN: + goto retry; + case SSL_ERROR_SYSCALL: + case SSL_ERROR_SSL: + goto complete; + default: + TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() " + "error %i. Closing fd.\n", sslerr); + goto complete; + } + goto complete; + +retry: + if (ctx->retries++ >= 50) { + TFE_LOG_ERROR(logger,"Failed to shutdown SSL connection cleanly: " + "Max retries reached. Closing fd.\n"); + goto complete; + } + ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); + if (ctx->ev) { + event_add(ctx->ev, &retry_delay); + return; + } + TFE_LOG_ERROR(logger,"Failed to shutdown SSL connection cleanly: " + "Cannot create event. Closing fd.\n"); + +complete: + ssl_stream_free(ctx->s_stream); + evutil_closesocket(fd); + ssl_shutdown_ctx_free(ctx); + +} + +/* + * Cleanly shutdown an SSL session on file descriptor fd using low-level + * file descriptor readiness events on event base evbase. + * Guarantees that SSL and the corresponding SSL_CTX are freed and the + * socket is closed, eventually, or in the case of fatal errors, immediately. + */ +void ssl_stream_free_and_close_fd(struct ssl_stream* s_stream, struct event_base *evbase, evutil_socket_t fd) +{ + struct ssl_shutdown_ctx *sslshutctx=NULL; + sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); + pxy_ssl_shutdown_cb(fd, 0, sslshutctx); + return; }