diff --git a/platform/include/internal/ssl_stream.h b/platform/include/internal/ssl_stream.h index b6407fc..e93d32e 100644 --- a/platform/include/internal/ssl_stream.h +++ b/platform/include/internal/ssl_stream.h @@ -17,7 +17,7 @@ void ssl_async_upstream_create(struct future* f, struct ssl_mgr* mgr, evutil_soc struct ssl_stream* ssl_downstream_create_result_release_stream(future_result_t* result); struct bufferevent* ssl_downstream_create_result_release_bev(future_result_t* result); -void ssl_async_downstream_create(struct future* f, struct ssl_mgr* mgr, struct ssl_stream* upstream, evutil_socket_t fd_downstream, struct event_base *evbase); +void ssl_async_downstream_create(struct future* f, struct ssl_mgr* mgr, struct ssl_stream* upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base *evbase); void ssl_stream_free_and_close_fd(struct ssl_stream* stream, struct event_base *evbase, evutil_socket_t fd); diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index c6f3565..9a597a4 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -1,3 +1,9 @@ +/* + * SSL stream implementation. + * Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise. + * Author: zhengchao@iie.ac.cn + * Create: 2018-8-17 + */ #include #include #include @@ -30,227 +36,303 @@ #include #include #include -#define SSL_EX_DATA_IDX_SSLMGR 0 -#define MAX_NET_RETRIES 50 -struct ssl_mgr -{ - int sslcomp; - int no_ssl2; - int no_ssl3; - int no_tls10; - int no_tls11; - int no_tls12; - CONST_SSL_METHOD *(*sslmethod)(void); //Parameter of SSL_CTX_new - int sslversion; - char ssl_session_context[8]; - int cache_slot_num; - int sess_expire_seconds; - struct sess_cache* down_sess_cache; - struct sess_cache* up_sess_cache; - char* default_ciphers[TFE_SYMBOL_MAX]; - DH * dh; - char * ecdhcurve; - char * crlurl; - uint8_t SSL_MODE_RELEASE_BUFFERS; - void *logger; -}; -struct ssl_stream -{ - enum tfe_conn_dir dir; - SSL * ssl; - struct ssl_mgr* ref_mgr; - char* sni; -}; -struct ssl_chello -{ - //client hello - int version; - char* sni; - char* cipher_suites; + +#define SSL_EX_DATA_IDX_SSLMGR 0 +#define MAX_NET_RETRIES 50 + + +struct ssl_mgr { + int sslcomp; + int no_ssl2; + int no_ssl3; + int no_tls10; + int no_tls11; + int no_tls12; + CONST_SSL_METHOD * (*sslmethod) (void); //Parameter of SSL_CTX_new + int sslversion; + char ssl_session_context[8]; + int cache_slot_num; + int sess_expire_seconds; + struct sess_cache * down_sess_cache; + struct sess_cache * up_sess_cache; + char * default_ciphers[TFE_SYMBOL_MAX]; + DH * dh; + char * ecdhcurve; + char * crlurl; + uint8_t SSL_MODE_RELEASE_BUFFERS; + void * logger; + struct cert_mgr * cert_mgr; }; -struct peek_client_hello_ctx -{ - unsigned char sni_peek_retries; /* max 64 SNI parse retries */ - struct event* ev; - struct event_base* evbase; + +struct ssl_stream { + enum tfe_conn_dir dir; + SSL * ssl; + struct ssl_mgr * mgr; + struct ssl_chello * client_hello; + struct ssl_stream* peer; }; -struct ssl_connect_origin_ctx -{ - struct bufferevent *bev; - struct ssl_stream* ssl; + + +struct ssl_chello { + //client hello + int version; + char * sni; + char * cipher_suites; +}; + + +struct peek_client_hello_ctx { + struct ssl_chello chello; + unsigned char sni_peek_retries; /* max 64 SNI parse retries */ + struct event * ev; + struct event_base * evbase; + void * logger; +}; + + +struct ssl_connect_origin_ctx { + struct bufferevent * bev; + struct ssl_stream * s_stream; + struct ssl_mgr * mgr; struct sockaddr addr; - socklen_t addrlen; + socklen_t addrlen; + void * logger; + + evutil_socket_t fd_upstream; + evutil_socket_t fd_downstream; + struct event_base * evbase; + struct future * f_peek_chello; }; + + +struct query_cert_ctx { + int keyring_id; + struct ssl_stream * origin_ssl; + X509 * origin_crt; + struct ssl_mgr * ssl_mgr; + evutil_socket_t fd_downstream; + struct event_base * evbase; + + struct future * f_query_cert; + struct bufferevent * bev_down; + struct ssl_stream* downstream; +}; + + /* * SSL shutdown context. */ struct ssl_shutdown_ctx { - struct ssl_stream* s_stream; - struct event_base *evbase; - struct event *ev; - unsigned int retries; + struct ssl_stream * s_stream; + struct event_base * evbase; + struct event * ev; + unsigned int retries; }; -static int parse_ssl_version(const char * version_str) + + +struct ssl_stream * ssl_stream_create(struct ssl_chello * client_hello, struct ssl_mgr * mgr, struct sockaddr * addr, + socklen_t addrlen) { + struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1); + s_stream->client_hello = client_hello; + s_stream->ssl = upstream_ssl_create(mgr, client_hello, addr, addrlen); + return s_stream; +} + + +static void ssl_stream_free(struct ssl_stream * s_stream) { - int sslversion=-1; + SSL_free(s_stream->ssl); + s_stream->ssl = NULL; + + if (s_stream->client_hello != NULL) { + ssl_free_chello(s_stream->client_hello;); + } + + s_stream->mgr = NULL; + free(s_stream); + return; +} + + +static int sslver_str2num(const char * version_str) +{ + int sslversion = -1; + assert(OPENSSL_VERSION_NUMBER >= 0x10100000L); + /* * Support for SSLv2 and the corresponding SSLv2_method(), * SSLv2_server_method() and SSLv2_client_method() functions were * removed in OpenSSL 1.1.0. */ - if (!strcmp(version_str, "ssl3")) - { - sslversion = SSL3_VERSION; + if (!strcmp(version_str, "ssl3")) { + sslversion = SSL3_VERSION; } - else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1")) - { - sslversion = TLS1_VERSION; + else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1")) { + sslversion = TLS1_VERSION; } - else if (!strcmp(version_str, "tls11")) - { - sslversion = TLS1_1_VERSION; - } - else if (!strcmp(version_str, "tls12")) - { - sslversion = TLS1_2_VERSION; - } - else - { - sslversion =-1; + else if (!strcmp(version_str, "tls11")) { + sslversion = TLS1_1_VERSION; } + else if (!strcmp(version_str, "tls12")) { + sslversion = TLS1_2_VERSION; + } + else { + sslversion = -1; + } + return sslversion; } -void ssl_manager_destroy(struct ssl_mgr* mgr) + + +void ssl_manager_destroy(struct ssl_mgr * mgr) { free(mgr); } -struct ssl_mgr* ssl_manager_init(const char* ini_profile, const char* section, void* logger) -{ - struct ssl_mgr* mgr=ALLOC(struct ssl_mgr, 1); - int ret=0,value=0; - char version_str[TFE_SYMBOL_MAX]; - mgr->logger=logger; - MESA_load_profile_string_def(ini_profile, section, "ssl_version", version_str, sizeof(version_str),"tls12"); - mgr->sslversion=parse_ssl_version(version_str); - if(mgr->sslversion<0) - { - TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s",version_str); + +struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, void * logger) { + struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1); + int ret = 0, value = 0; + char version_str[TFE_SYMBOL_MAX]; + mgr->logger = logger; + MESA_load_profile_string_def(ini_profile, section, "ssl_version", version_str, sizeof(version_str), "tls12"); + mgr->sslversion = sslver_str2num(version_str); + + if (mgr->sslversion < 0) { + TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str); goto error_out; } - //tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0. - mgr->sslmethod=TLS_method; - MESA_load_profile_int_def(ini_profile, section, "ssl_compression", &(mgr->sslcomp), 1); - MESA_load_profile_int_def(ini_profile, section, "no_ssl2", &(mgr->no_ssl2), 1); - MESA_load_profile_int_def(ini_profile, section, "no_ssl3", &(mgr->no_ssl3), 1); - MESA_load_profile_int_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 1); - MESA_load_profile_int_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0); - MESA_load_profile_int_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0); - MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", &(mgr->cache_slot_num), 4*1024*1024); - MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", &(mgr->sess_expire_seconds), 30*60); - - mgr->up_sess_cache=ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds,CONN_DIR_UPSTREAM); - mgr->down_sess_cache=ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds,CONN_DIR_DOWNSTREAM); - memcpy(mgr->ssl_session_context,"mesa-tfe",sizeof(mgr->ssl_session_context)); + + //tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0. + mgr->sslmethod = TLS_method; + MESA_load_profile_int_def(ini_profile, section, "ssl_compression", & (mgr->sslcomp), 1); + MESA_load_profile_int_def(ini_profile, section, "no_ssl2", & (mgr->no_ssl2), 1); + MESA_load_profile_int_def(ini_profile, section, "no_ssl3", & (mgr->no_ssl3), 1); + MESA_load_profile_int_def(ini_profile, section, "no_tls10", & (mgr->no_tls10), 1); + MESA_load_profile_int_def(ini_profile, section, "no_tls11", & (mgr->no_tls11), 0); + MESA_load_profile_int_def(ini_profile, section, "no_tls12", & (mgr->no_tls12), 0); + MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", & (mgr->cache_slot_num), + 4 * 1024 * 1024); + MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", & (mgr->sess_expire_seconds), 30 * 60); + + mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); + mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); + mgr->cert_mgr = cert_mgr_init(ini_profile, section); + + if (mgr->cert_mgr == NULL) { + TFE_LOG_ERROR(logger, "Certificate Manager initiate failed."); + goto error_out; + } + + + memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context)); return mgr; + + + + + + + + + + error_out: + ssl_manager_destroy(mgr); return NULL; -} - - -void peek_client_hello_ctx_free(void* ctx) +} void peek_client_hello_ctx_free(void * ctx) { - struct peek_client_hello_ctx * _ctx=(struct peek_client_hello_ctx *)ctx; + struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *) + ctx; event_free(_ctx->ev); - _ctx->ev = NULL; - free(_ctx->sni); - _ctx->sni=NULL; + _ctx->ev = NULL; + free(_ctx->chello.sni); + _ctx->chello.sni = NULL; + free(_ctx->chello.cipher_suites); + _ctx->chello.cipher_suites = NULL; free(_ctx); return; } -struct ssl_chello* ssl_peek_result_release_chello(future_result_t* result) -{ - struct ssl_chello* p=(struct ssl_chello* )result, *copy=NULL; - copy=ALLOC(struct ssl_chello*,1); - if(p!=NULL) - { - copy->sni=tfe_strdup(p->sni); - copy->cipher_suites=tfe_strdup(p->cipher_suites); - copy->version=p->version; + + +struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) { + struct ssl_chello * p = (struct ssl_chello *) + result, *copy = NULL; + copy = ALLOC(struct ssl_chello *, 1); + + if (p != NULL) { + copy->sni = tfe_strdup(p->sni); + copy->cipher_suites = tfe_strdup(p->cipher_suites); + copy->version = p->version; } + + return copy; -} -void ssl_free_chello(struct ssl_chello* p) +} void ssl_free_chello(struct ssl_chello * p) { - if(p==NULL) - { + if (p == NULL) { return; } + free(p->sni); - p->sni=NULL; + p->sni = NULL; free(p->cipher_suites); - p->cipher_suites=NULL; - p->cipher_suites=NULL; + p->cipher_suites = NULL; free(p); return; } + static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) { - struct promise* promise=(struct promise*)arg; - struct peek_client_hello_ctx* ctx= (struct peek_client_hello_ctx*)promise_get_ctx(promise); - struct ssl_chello *result=NULL; - const char* reason_too_many_retries="too many tries"; - const char* reason_see_no_client_hello="see no client hello"; - const char* reason=NULL; - memset(&result, 0, sizeof(result)); - char* sni=NULL; - unsigned char buf[1024]; - ssize_t n=0; - const unsigned char * chello=NULL; - int rv=0; + struct promise * promise = (struct promise *)arg; + struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *)promise_get_ctx(promise); + const char * reason_too_many_retries = "too many tries"; + const char * reason_see_no_client_hello = "see no client hello"; + const char * reason = NULL; + char * sni = NULL; + unsigned char buf[1024]; + ssize_t n = 0; + const unsigned char * chello = NULL; + int rv = 0; - n = recv(fd, buf, sizeof(buf), MSG_PEEK); - if (n == -1) - { - TFE_LOG_ERROR("Error peeking on fd, aborting connection\n"); + n = recv(fd, buf, sizeof(buf), MSG_PEEK); + + if (n == -1) { + TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection\n"); goto failed; } - - if (n == 0) - { + + if (n == 0) { goto failed; } - result=ssl_get_peek_result(NULL); + //todo: parse version and cipher suites. //or we should use sni proxy instead? https://github.com/dlundquist/sniproxy/blob/master/src/tls.c - rv = ssl_tls_clienthello_parse(buf, n, 0, &chello, &result->sni); - if(rv==0) - { + rv = ssl_tls_clienthello_parse(buf, n, 0, &chello, & (ctx->chello.sni)); + + if (rv == 0) { promise_dettach_ctx(promise); - promise_success(promise, result); - ssl_free_chello(result); + promise_success(promise, & (ctx->chello)); peek_client_hello_ctx_free(ctx); } - else - { - if(!chello) - { - TFE_LOG_ERROR("Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); - reason="see no client hello"; - goto failed; - } - - if(ctx->sni_peek_retries++ > MAX_NET_RETRIES) - { - TFE_LOG_ERROR("Peek failed due to too many retries\n"); - reason="too many peek retries"; + else { + if (!chello) { + TFE_LOG_ERROR(ctx->logger, "Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); + reason = "see no client hello"; goto failed; } + + if (ctx->sni_peek_retries++ > MAX_NET_RETRIES) { + TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n"); + reason = "too many peek retries"; + goto failed; + } + /* ssl_tls_clienthello_parse indicates that we * should retry later when we have more data, and we * haven't reached the maximum retry count yet. @@ -258,121 +340,143 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) * order to prevent busy looping over the read event. * Because we only peeked at the pending bytes and * never actually read them, fd is still ready for - * reading now. We use 25 * 0.2 s = 5 s timeout. */ - struct timeval retry_delay = {0, 100}; + * reading now. We use 25 * 0.2 s = 5 s timeout. */ + struct timeval retry_delay = { + 0, 100 + }; + + event_free(ctx->ev); - ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); - assert(ctx->ev!=NULL); + ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); + assert(ctx->ev != NULL); event_add(ctx->ev, &retry_delay); } + return; + failed: promise_dettach_ctx(promise); - promise_failed(promise,FUTURE_ERROR_EXCEPTION,reason); + promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason); peek_client_hello_ctx_free(ctx); ssl_free_chello(result); return; } -void ssl_async_peek_client_hello(struct future* future, evutil_socket_t fd, struct event_base *evbase) +static void ssl_async_peek_client_hello(struct future * future, evutil_socket_t fd, struct event_base * evbase, + void * logger) { - struct event * ev=NULL; - struct promise* p=future_to_promise(future); - - struct peek_client_hello_ctx* ctx=ALLOC(struct peek_client_hello_ctx, 1); - ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); + struct event * ev = NULL; + struct promise * p = future_to_promise(future); + struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1); + ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); + ctx->logger = logger; event_add(evbase, NULL); promise_set_ctx(p, ctx, peek_client_hello_ctx_free); return; } + + /* * Create new SSL context for outgoing connections to the original destination. * If hostname sni is provided, use it for Server Name Indication. */ - -static SSL* upstream_ssl_create(struct ssl_mgr* mgr, const struct ssl_chello* chello, struct sockaddr* addr, socklen_t addrlen) +static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, struct sockaddr * addr, + socklen_t addrlen) { - SSL_CTX* sslctx=NULL; - SSL* ssl=NULL; - SSL_SESSION* sess=NULL; - + SSL_CTX * sslctx = NULL; + SSL * ssl = NULL; + SSL_SESSION * sess = NULL; - sslctx = SSL_CTX_new(mgr->sslmethod()); + + sslctx = SSL_CTX_new(mgr->sslmethod()); sslctx_set_opts(sslctx, mgr); -#if OPENSSL_VERSION_NUMBER >= 0x10100000L +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + if (mgr->sslversion) { if (SSL_CTX_set_min_proto_version(sslctx, chello->version) == 0 || - SSL_CTX_set_max_proto_version(sslctx, chello->version) == 0) { + SSL_CTX_set_max_proto_version(sslctx, chello->version) == 0) { SSL_CTX_free(sslctx); return NULL; } } + #endif /* OPENSSL_VERSION_NUMBER >= 0x10100000L */ SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); - ssl = SSL_new(sslctx); - SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ - if (!ssl) - { + ssl = SSL_new(sslctx); + SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ + + if (!ssl) { return NULL; } + #ifndef OPENSSL_NO_TLSEXT - if (chello->sni) - { + + if (chello->sni) { SSL_set_tlsext_host_name(ssl, sni); } + #endif /* !OPENSSL_NO_TLSEXT */ #ifdef SSL_MODE_RELEASE_BUFFERS + /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); #endif /* SSL_MODE_RELEASE_BUFFERS */ //Todo: Refactor below session resumption. + /* session resuming based on remote endpoint address and port */ - sess = up_session_get(mgr->up_sess_cache, addr, - addrlen, chello->sni); /* new sess inst */ - if (sess) - { - SSL_set_session(ssl, sess); /* increments sess refcount */ + sess = up_session_get(mgr->up_sess_cache, addr, + addrlen, chello->sni); /* new sess inst */ + + if (sess) { + SSL_set_session(ssl, sess); /* increments sess refcount */ SSL_SESSION_free(sess); } return ssl; } -void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx* ctx) + +void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx) { - if(ctx->ssl!=NULL) - { - ssl_stream_shutdown(ctx->bev,ctx->ssl); - free(ctx->ssl); + if (ctx->s_stream != NULL) { + ssl_stream_free(ctx->s_stream); } - if(ctx->bev!=NULL) - { - bufferevent_free(ctx->bev); - ctx->bev=NULL; + + if (ctx->bev != NULL) { + bufferevent_free(ctx->bev); + ctx->bev = NULL; } + + if (ctx->f_peek_chello != NULL) { + future_destroy(ctx->f_peek_chello); + ctx->f_peek_chello = NULL; + } + free(ctx); return; } -struct ssl_stream* ssl_conn_origin_result_release_stream(future_result_t* result) -{ - struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx* )result; - struct ssl_stream* ret=ctx->ssl; - ctx->ssl=NULL; //giveup ownership + + +struct ssl_stream * ssl_conn_origin_result_release_stream(future_result_t * result) { + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)result; + struct ssl_stream * ret = ctx->s_stream; + ctx->s_stream = NULL; //giveup ownership return ret; } -struct bufferevent* ssl_conn_origin_result_release_bev(future_result_t* result) -{ - struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx* )result; - struct bufferevent * ret=ctx->bev; - ctx->bev=NULL; //giveup ownership + + +struct bufferevent * ssl_conn_origin_result_release_bev(future_result_t * result) { + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)result; + struct bufferevent * ret = ctx->bev; + ctx->bev = NULL; //giveup ownership return ret; } @@ -383,54 +487,83 @@ struct bufferevent* ssl_conn_origin_result_release_bev(future_result_t* result) */ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg) { - struct promise* promise=(struct promise*)arg; - struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx*)promise_get_ctx(promise); - SSL_SESSION * to_cached=NULL; - struct tfe_stream_private* _stream = (struct tfe_stream_private*)arg; - - if (events & BEV_EVENT_ERROR) - { - promise_failed(promise, FUTURE_ERROR_EXCEPTION, "ssl connect failed"); - } - else - { - if (events & BEV_EVENT_CONNECTED) - { - bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL);//leave a clean bev for on_success - to_cached=SSL_get0_session(ctx->ssl->ssl); - up_session_set(ctx->ssl->ref_mgr->up_sess_cache, &(ctx->addr), ctx->addrlen, ctx->ssl->sni, to_cached); - promise_dettach_ctx(promise); - promise_success(promise, ctx); - ctx->bev=NULL; - } - else - { + struct promise * promise = (struct promise *)arg; + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)promise_dettach_ctx(promise); + + struct ssl_stream * s_stream = ctx->s_stream; + SSL_SESSION * ssl_sess = NULL; + + if (events & BEV_EVENT_ERROR) { + promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to orignal server failed."); + } + else { + if (events & BEV_EVENT_CONNECTED) { + bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success + ssl_sess = SSL_get0_session(s_stream->ssl); + up_session_set(s_stream->mgr->up_sess_cache, & (ctx->addr), ctx->addrlen, s_stream->client_hello.sni, + ssl_sess); + promise_success(promise, ctx); + } + else { assert(0); } } + ssl_connect_origin_ctx_free(ctx); return } -void ssl_async_upstream_create(struct future* f, struct ssl_mgr* mgr, const struct ssl_chello* chello, evutil_socket_t fd, struct event_base *evbase) -{ - struct promise* p=future_to_promise(future); - struct ssl_connect_origin_ctx* ctx=ALLOC(struct ssl_connect_origin_ctx, 1); - int ret=0; - struct sockaddr addr; - socklen_t addrlen; - ret=getpeername(fd, &(ctx->addr), &(ctx->addrlen)); - assert(ret==0); - ctx->ssl=ALLOC(struct ssl_stream,1); - ctx->ssl->ref_mgr=mgr; - ctx->ssl->dir=CONN_DIR_UPSTREAM; - ctx->ssl->sni=tfe_strdup(chello->sni); - ctx->ssl->ssl=upstream_ssl_create(mgr, chello, &(ctx->addr), ctx->addrlen); - ctx->bev = bufferevent_openssl_socket_new(evbase, fd, ctx->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); - promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free); + +static void peek_chello_on_succ(future_result_t * result, void * user) +{ + struct promise* p= (struct promise*)user; + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); + + struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. + ctx->s_stream = ssl_stream_create(chello, ctx->mgr, & (ctx->addr), ctx->addrlen); + ctx->bev = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_upstream, ctx->s_stream, + BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1); - bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, p); - bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); + bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, ctx); + bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only + future_destroy(ctx->f_peek_chello); + ctx->f_peek_chello = NULL; + return; +} + + +static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user) +{ + struct promise* p= (struct promise*)user; + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); + + promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream."); + ssl_connect_origin_ctx_free(ctx); + return; +} + + +extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, + evutil_socket_t fd_downstream, struct event_base * evbase) +{ + struct promise * p = future_to_promise(f); + struct ssl_connect_origin_ctx * ctx = ALLOC(struct ssl_connect_origin_ctx, 1); + int ret = 0; + + struct sockaddr addr; + socklen_t addrlen; + + ret = getpeername(fd_downstream, & (ctx->addr), & (ctx->addrlen)); + assert(ret == 0); + ctx->fd_downstream = fd_downstream; + ctx->fd_upstream = fd_upstream; + ctx->evbase = evbase; + ctx->mgr = mgr; + promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free); + + ctx->f_peek_chello = future_create(peek_chello_on_succ, peek_chello_on_fail, p); + ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); + return; } @@ -441,117 +574,142 @@ void ssl_async_upstream_create(struct future* f, struct ssl_mgr* mgr, const stru * the refcount decrementing. In other words, return 0 if we did not * keep a pointer to the object (which we never do here). */ - -static int -ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) +static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) { - struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); + struct ssl_mgr * mgr = (struct ssl_mgr *) + SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); #ifdef HAVE_SSLV2 + /* Session resumption seems to fail for SSLv2 with protocol * parsing errors, so we disable caching for SSLv2. */ if (SSL_version(ssl) == SSL2_VERSION) { return 0; } + #endif /* HAVE_SSLV2 */ - if (sess) - { + + if (sess) { down_session_set(mgr->down_sess_cache, sess); } + return 0; } + /* * Called by OpenSSL when a src SSL session should be removed. * OpenSSL calls SSL_SESSION_free() after calling the callback; * we do not need to free the reference here. */ -static void -ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) +static void ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) { - struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); - assert(mgr!=NULL); - if (sess) - { - down_session_del(mgr->down_sess_cache, sess); + struct ssl_mgr * mgr = (struct ssl_mgr *) + SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); + assert(mgr != NULL); + + if (sess) { + down_session_del(mgr->down_sess_cache, sess) ; } + return; } + /* * Called by OpenSSL when a src SSL session is requested by the client. - OPENSSL_VERSION_NUMBER >= 0x10100000L required. + OPENSSL_VERSION_NUMBER >= 0x10100000L required. */ -static SSL_SESSION * -ossl_sessget_cb(UNUSED SSL *ssl, const unsigned char *id, int idlen, int *copy) +static SSL_SESSION * ossl_sessget_cb(UNUSED SSL * ssl, const unsigned char * id, int idlen, int * copy) { - struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); + struct ssl_mgr * mgr = (struct ssl_mgr *) + SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); SSL_SESSION * sess; - *copy = 0; /* SSL should not increment reference count of session */ - sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache,id, idlen); + *copy = 0; /* SSL should not increment reference count of session */ + sess = (SSL_SESSION *) + down_session_get(mgr->down_sess_cache, id, idlen); return sess; } + /* * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX. */ -static void -sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr* mgr) +static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr) { SSL_CTX_set_options(sslctx, SSL_OP_ALL); + #ifdef SSL_OP_TLS_ROLLBACK_BUG SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG); #endif /* SSL_OP_TLS_ROLLBACK_BUG */ + #ifdef SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); #endif /* SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION */ + #ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); #endif /* SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS */ + #ifdef SSL_OP_NO_TICKET SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET); #endif /* SSL_OP_NO_TICKET */ #ifdef SSL_OP_NO_SSLv2 + #ifdef HAVE_SSLV2 + if (mgr->no_ssl2) { #endif /* HAVE_SSLV2 */ - SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); + + SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); + #ifdef HAVE_SSLV2 } + #endif /* HAVE_SSLV2 */ + #endif /* !SSL_OP_NO_SSLv2 */ + #ifdef HAVE_SSLV3 - if (mgr->no_ssl3) - { + + if (mgr->no_ssl3) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); } + #endif /* HAVE_SSLV3 */ + #ifdef HAVE_TLSV10 - if (mgr->no_tls10) - { + + if (mgr->no_tls10) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1); } + #endif /* HAVE_TLSV10 */ + #ifdef HAVE_TLSV11 - if (mgr->no_tls11) - { + + if (mgr->no_tls11) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1); } + #endif /* HAVE_TLSV11 */ + #ifdef HAVE_TLSV12 - if (mgr->no_tls12) - { + + if (mgr->no_tls12) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2); } + #endif /* HAVE_TLSV12 */ #ifdef SSL_OP_NO_COMPRESSION - if (!mgr->sslcomp) - { + + if (!mgr->sslcomp) { SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION); } + #endif /* SSL_OP_NO_COMPRESSION */ SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); @@ -562,106 +720,180 @@ sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr* mgr) * Create and set up a new SSL_CTX instance for terminating SSL. * Set up all the necessary callbacks, the cert, the cert chain and key. */ -static SSL_CTX * down_sslctx_create(struct ssl_mgr* mgr, struct cert* crt) +static SSL_CTX * down_sslctx_create(struct ssl_mgr * mgr, struct cert * crt) { - SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); - if (!sslctx) - return NULL; + SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); - sslctx_set_opts(sslctx, mgr); + if (!sslctx) + return NULL; -//TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L - if (mgr->sslversion) { - if (SSL_CTX_set_min_proto_version(sslctx,mgr->sslversion) == 0 || - SSL_CTX_set_max_proto_version(sslctx, mgr->sslversion) == 0) { - SSL_CTX_free(sslctx); - return NULL; - } - } + sslctx_set_opts(sslctx, mgr); + + //TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L + if (mgr->sslversion) { + if (SSL_CTX_set_min_proto_version(sslctx, mgr->sslversion) == 0 || + SSL_CTX_set_max_proto_version(sslctx, mgr->sslversion) == 0) { + SSL_CTX_free(sslctx); + return NULL; + } + } + + SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); + SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); + SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); + SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); + SSL_CTX_set_session_id_context(sslctx, (void *) (mgr->ssl_session_context), + sizeof(mgr->ssl_session_context)); - SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); - SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); - SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); - SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | - SSL_SESS_CACHE_NO_INTERNAL); - SSL_CTX_set_session_id_context(sslctx, (void *)(mgr->ssl_session_context), - sizeof(mgr->ssl_session_context)); #ifndef OPENSSL_NO_DH - if (mgr->dh) - { - SSL_CTX_set_tmp_dh(sslctx, mgr->dh); - } - else - { - SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); - } + + if (mgr->dh) { + SSL_CTX_set_tmp_dh(sslctx, mgr->dh); + } + else { + SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); + } + #endif /* !OPENSSL_NO_DH */ + #ifndef OPENSSL_NO_ECDH - if (mgr->ecdhcurve) - { - EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); - SSL_CTX_set_tmp_ecdh(sslctx, ecdh); - EC_KEY_free(ecdh); - } - else - { - EC_KEY * ecdh = ssl_ec_by_name(NULL); - SSL_CTX_set_tmp_ecdh(sslctx, ecdh); - EC_KEY_free(ecdh); - } + + if (mgr->ecdhcurve) { + EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); + SSL_CTX_set_tmp_ecdh(sslctx, ecdh); + EC_KEY_free(ecdh); + } + else { + EC_KEY * ecdh = ssl_ec_by_name(NULL); + SSL_CTX_set_tmp_ecdh(sslctx, ecdh); + EC_KEY_free(ecdh); + } + #endif /* !OPENSSL_NO_ECDH */ - SSL_CTX_use_certificate(sslctx, crt->crt); - SSL_CTX_use_PrivateKey(sslctx, crt->key); - for (int i = 0; i < sk_X509_num(crt->chain); i++) - { - X509 * c = sk_X509_value(crt->chain, i); - ssl_x509_refcount_inc(c); /* next call consumes a reference */ - SSL_CTX_add_extra_chain_cert(sslctx, c); - } - return sslctx; + + SSL_CTX_use_certificate(sslctx, crt->crt); + SSL_CTX_use_PrivateKey(sslctx, crt->key); + + for (int i = 0; i < sk_X509_num(crt->chain); i++) { + X509 * c = sk_X509_value(crt->chain, i); + ssl_x509_refcount_inc(c); /* next call consumes a reference */ + SSL_CTX_add_extra_chain_cert(sslctx, c); + } + + return sslctx; +} + + +void query_cert_ctx_free(struct query_cert_ctx * ctx) +{ + X509_free(ctx->origin_crt); + + if (ctx->f_query_cert != NULL) { + future_destroy(ctx->f_query_cert); + ctx->f_query_cert = NULL; + } + if(ctx->bev_down!=NULL) + { + bufferevent_free(ctx->bev_down); + } + if(ctx->downstream!=NULL) + { + ssl_stream_free(ctx->downstream); + } + return; +} + +struct ssl_stream* ssl_downstream_create_result_release_stream(future_result_t* result) +{ + struct query_cert_ctx * ctx = (struct query_cert_ctx *)result; + struct ssl_stream* ret=ctx->downstream; + ctx->downstream=NULL; + return ret; +} +struct bufferevent* ssl_downstream_create_result_release_bev(future_result_t* result) +{ + struct query_cert_ctx * ctx = (struct query_cert_ctx *)result; + struct bufferevent* ret=ctx->bev_down; + ctx->bev_down=NULL; + return ret; +} + +void query_cert_on_succ(void * result, void * user) +{ + struct promise* p=(struct promise*)user; + struct query_cert_ctx * ctx = (struct query_cert_ctx *)promise_dettach_ctx(promise); + + struct ssl_stream * downstream = NULL; + struct cert* crt=NULL; + SSL_CTX * sslctx=NULL; + + struct ssl_mgr* mgr=ctx->ssl_mgr; + + future_destroy(ctx->f_query_cert); + ctx->f_query_cert=NULL; + + crt=cert_mgr_query_result_release_cert(result); + sslctx = down_sslctx_create(mgr, crt); + assert(sslctx!=NULL); + + downstream = ALLOC(struct ssl_stream, 1); + downstream->mgr = mgr; + downstream->dir = CONN_DIR_DOWNSTREAM; + downstream->ssl = SSL_new(sslctx); + downstream->peer = ctx->origin_ssl; + ctx->origin_ssl->peer= downstream; + int ret = SSL_set_ex_data(downstream->ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); + assert(ret == 1); + SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ + + if (mgr->SSL_MODE_RELEASE_BUFFERS == 1) { + /* lower memory footprint for idle connections */ + SSL_set_mode(downstream->ssl, SSL_get_mode(downstream->ssl) | SSL_MODE_RELEASE_BUFFERS); + } + ctx->downstream = downstream; + ctx->bev_down = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl, + BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); + bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); + + promise_success(p, ctx); + cert_mgr_free_cert(crt); + query_cert_ctx_free(ctx); + return; +} + + +void query_cert_on_fail(enum e_future_error error, const char * what, void * user) +{ + struct promise* p=(struct promise*)user; + struct query_cert_ctx * ctx = (struct query_cert_ctx *)promise_dettach_ctx(promise); + promise_failed(p, error, what); + query_cert_ctx_free(ctx); + return; } /* - * Create new SSL context for the incoming connection, based on the a designate cert - * Returns NULL when openssl error. + * Create a SSL stream for the incoming connection, based on the a upstream. */ -void ssl_async_downstream_create(struct future* f, struct ssl_mgr* mgr, struct ssl_chello* hello, evutil_socket_t fd, struct event_base *evbase) +void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base * evbase) { - struct cert* crt; - struct ssl_stream* p=NULL; - if(crt==NULL) - { - goto error_out; - } - p=ALLOC(struct ssl_stream, 1); - p->ref_mgr=mgr; - p->dir=CONN_DIR_DOWNSTREAM; - //could be optimized by store the sslctx. - SSL_CTX * sslctx = down_sslctx_create(mgr,crt); - cert_mgr_free_cert(crt); - if (!sslctx) - { - goto error_out; - } - p->ssl = SSL_new(sslctx); - int ret=SSL_set_ex_data(p->ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); - assert(ret==1); - SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ - if (!p->ssl) - { - goto error_out; - } - if(mgr->SSL_MODE_RELEASE_BUFFERS==1) - { - /* lower memory footprint for idle connections */ - SSL_set_mode(p->ssl, SSL_get_mode(p->ssl) | SSL_MODE_RELEASE_BUFFERS); - } - p->sni=tfe_strdup(hello->sni); - return p; -error_out: - free(p); - return NULL; + + assert(upstream->dir == CONN_DIR_UPSTREAM); + + struct query_cert_ctx * ctx = ALLOC(struct query_cert_ctx, 1); + ctx->keyring_id = keyring_id; + ctx->origin_ssl = upstream; + ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl); + ctx->ssl_mgr = mgr; + ctx->fd_downstream = fd_downstream; + ctx->evbase = evbase; + + struct promise* p = future_to_promise(f); + promise_set_ctx(p, ctx, query_cert_ctx_free); + //todo: verify original certificate. + ctx->f_query_cert = future_create(query_cert_on_succ, query_cert_on_fail, p); + cert_mgr_async_query(ctx->f_query_cert, mgr->cert_mgr, keyring_id, ctx->origin_crt, evbase); + return; } @@ -672,48 +904,43 @@ error_out: * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE * may or may not work. */ -static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream* s_stream, struct event_base *evbase) -{ - struct ssl_shutdown_ctx *ctx = ALLOC(struct ssl_shutdown_ctx,1); +static + + +struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase) { + struct ssl_shutdown_ctx * ctx = ALLOC(struct ssl_shutdown_ctx, 1); ctx->evbase = evbase; ctx->s_stream = s_stream; ctx->ev = NULL; ctx->retries = 0; return ctx; -} - -static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx *ctx) +} static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx) { free(ctx); } -static void ssl_stream_free(struct ssl_stream* s_stream) -{ - SSL_free(s_stream->ssl); - s_stream->ssl=NULL; - free(s_stream->sni); - s_stream->sni=NULL; - s_stream->ref_mgr=NULL - free(s_stream); - return; -} + + /* * The shutdown socket event handler. This is either * scheduled as a timeout-only event, or as a fd read or * fd write event, depending on whether SSL_shutdown() * indicates it needs read or write on the socket. */ -static void -pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void *arg) +static void pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void * arg) { - struct ssl_shutdown_ctx *ctx = (struct ssl_shutdown_ctx *)arg; - struct timeval retry_delay = {0, 100}; - void *logger=ctx->s_stream->ref_mgr->logger; - short want = 0; - int rv=0, sslerr=0; + struct ssl_shutdown_ctx * ctx = (struct ssl_shutdown_ctx *) arg; + + struct timeval retry_delay = { + 0, 100 + }; + + void * logger = ctx->s_stream->mgr->logger; + short want = 0; + int rv = 0, sslerr = 0; if (ctx->ev) { event_free(ctx->ev); - ctx->ev = NULL; + ctx->ev = NULL; } /* @@ -725,13 +952,17 @@ pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void *arg) * This is a good collection of recent and relevant documents: * http://bugs.python.org/issue8108 */ - rv = SSL_shutdown(ctx->s_stream->ssl); + rv = SSL_shutdown(ctx->s_stream->ssl); + if (rv == 1) goto complete; + if (rv != -1) { goto retry; } - switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) { + + switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) + { case SSL_ERROR_WANT_READ: want = EV_READ; goto retry; @@ -745,24 +976,30 @@ pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void *arg) goto complete; default: TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() " - "error %i. Closing fd.\n", sslerr); + "error %i. Closing fd.\n", sslerr); goto complete; } + goto complete; retry: + if (ctx->retries++ >= MAX_NET_RETRIES) { - TFE_LOG_ERROR(logger,"Failed to shutdown SSL connection cleanly: " - "Max retries reached. Closing fd.\n"); + TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " + "Max retries reached. Closing fd.\n"); goto complete; } - ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); + + ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); + if (ctx->ev) { event_add(ctx->ev, &retry_delay); return; } - TFE_LOG_ERROR(logger,"Failed to shutdown SSL connection cleanly: " - "Cannot create event. Closing fd.\n"); + + TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " + "Cannot create event. Closing fd.\n"); + complete: ssl_stream_free(ctx->s_stream); @@ -771,16 +1008,17 @@ complete: } + /* * Cleanly shutdown an SSL session on file descriptor fd using low-level * file descriptor readiness events on event base evbase. * Guarantees that SSL and the corresponding SSL_CTX are freed and the * socket is closed, eventually, or in the case of fatal errors, immediately. */ -void ssl_stream_free_and_close_fd(struct ssl_stream* s_stream, struct event_base *evbase, evutil_socket_t fd) +void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, evutil_socket_t fd) { - struct ssl_shutdown_ctx *sslshutctx=NULL; - sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); + struct ssl_shutdown_ctx * sslshutctx = NULL; + sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); pxy_ssl_shutdown_cb(fd, 0, sslshutctx); return; }