diff --git a/platform/include/internal/cert.h b/platform/include/internal/cert.h deleted file mode 100644 index e392297..0000000 --- a/platform/include/internal/cert.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef CERT_H -#define CERT_H - -#include -#include - -struct cert -{ - EVP_PKEY *key; - X509 *crt; - STACK_OF(X509) * chain; -}; -struct cert_mgr; -struct cert_mgr * cert_mgr_init(const char * profile, const char* section); -struct cert* cert_mgr_query_result_release_cert(future_result_t* result); -void cert_mgr_free_cert(struct cert* cert); -void cert_mgr_async_query(struct future * future, struct cert_mgr * mgr, int keyring_id, - X509 * origin_cert, struct event_base * evbase); -#endif /* !CERT_H */ diff --git a/platform/include/internal/key_keeper.h b/platform/include/internal/key_keeper.h new file mode 100644 index 0000000..5b3b77e --- /dev/null +++ b/platform/include/internal/key_keeper.h @@ -0,0 +1,21 @@ +#ifndef CERT_H +#define CERT_H + +#include +#include + +struct keyring +{ + EVP_PKEY *key; + X509 *cert; + STACK_OF(X509) * chain; +}; +struct key_keeper; +struct key_keeper * key_keeper_init(const char * profile, const char* section, void* logger); +struct key_keeper * key_keeper_destroy(struct key_keeper *keeper); + +struct keyring* key_keeper_release_cert(future_result_t* result); +void key_keeper_free_keyring(struct keyring* cert); +void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, int keyring_id, + X509 * origin_cert, int is_cert_valid, struct event_base * evbase); +#endif /* !CERT_H */ diff --git a/platform/include/internal/ssl_stream.h b/platform/include/internal/ssl_stream.h index b6407fc..9242d00 100644 --- a/platform/include/internal/ssl_stream.h +++ b/platform/include/internal/ssl_stream.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include @@ -17,7 +16,7 @@ void ssl_async_upstream_create(struct future* f, struct ssl_mgr* mgr, evutil_soc struct ssl_stream* ssl_downstream_create_result_release_stream(future_result_t* result); struct bufferevent* ssl_downstream_create_result_release_bev(future_result_t* result); -void ssl_async_downstream_create(struct future* f, struct ssl_mgr* mgr, struct ssl_stream* upstream, evutil_socket_t fd_downstream, struct event_base *evbase); +void ssl_async_downstream_create(struct future* f, struct ssl_mgr* mgr, struct ssl_stream* upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base *evbase); void ssl_stream_free_and_close_fd(struct ssl_stream* stream, struct event_base *evbase, evutil_socket_t fd); diff --git a/platform/src/cert.cpp b/platform/src/key_keeper.cpp similarity index 77% rename from platform/src/cert.cpp rename to platform/src/key_keeper.cpp index 62e5974..6d8ca31 100644 --- a/platform/src/cert.cpp +++ b/platform/src/key_keeper.cpp @@ -1,21 +1,21 @@ -#include +#include #include #include -struct cert_mgr +struct key_keeper { }; -struct tfe_cert_private +struct key_keeper_private { - struct cert head; + struct keyring head; pthread_mutex_t mutex; size_t references; }; #if 0 /* - * Certificate, including private key and cert chain. + * Certificate, including private key and keyring chain. */ cert_t * @@ -175,7 +175,7 @@ cert_set_chain(cert_t *c, STACK_OF(X509) *chain) } /* - * Free cert including internal objects. + * Free keyring including internal objects. */ void cert_free(cert_t *c) @@ -200,22 +200,22 @@ cert_free(cert_t *c) free(c); } -struct cert_mgr* cert_mgr_init(const char* profile) +struct key_keeper* key_keeper_init(const char* profile) { } -void cert_mgr_async_get(struct future* future, struct cert_mgr* mgr, int keyring_id, X509* origin_cert, struct event_base* evbase) +void cert_mgr_async_get(struct future* future, struct key_keeper* mgr, int keyring_id, X509* origin_cert, struct event_base* evbase) { X509* orig_cert=SSL_get_peer_certificate(origssl); //todo: need implement - cert_t * cert = NULL; + cert_t * keyring = NULL; if (opts->tgcrtdir) { if (ctx->sni) { - cert = (cert_t *) cachemgr_tgcrt_get(ctx->sni); - if (!cert) + keyring = (cert_t *) cachemgr_tgcrt_get(ctx->sni); + if (!keyring) { char * wildcarded = ssl_wildcardify(ctx->sni); if (!wildcarded) @@ -223,12 +223,12 @@ void cert_mgr_async_get(struct future* future, struct cert_mgr* mgr, int keyring ctx->enomem = 1; return NULL; } - cert = (cert_t *) cachemgr_tgcrt_get(wildcarded); + keyring = (cert_t *) cachemgr_tgcrt_get(wildcarded); free(wildcarded); } - if (cert && OPTS_DEBUG(ctx->opts)) + if (keyring && OPTS_DEBUG(ctx->opts)) { - log_dbg_printf("Target cert by SNI\n"); + log_dbg_printf("Target keyring by SNI\n"); } } else if (ctx->origcrt) @@ -236,11 +236,11 @@ void cert_mgr_async_get(struct future* future, struct cert_mgr* mgr, int keyring char ** names = ssl_x509_names(ctx->origcrt); for (char ** p = names; *p; p++) { - if (!cert) + if (!keyring) { - cert = (cert_t *) cachemgr_tgcrt_get(*p); + keyring = (cert_t *) cachemgr_tgcrt_get(*p); } - if (!cert) + if (!keyring) { char * wildcarded = ssl_wildcardify(*p); if (!wildcarded) @@ -249,7 +249,7 @@ void cert_mgr_async_get(struct future* future, struct cert_mgr* mgr, int keyring } else { - cert = (cert_t *) (wildcarded); + keyring = (cert_t *) (wildcarded); free(wildcarded); } } @@ -260,41 +260,41 @@ void cert_mgr_async_get(struct future* future, struct cert_mgr* mgr, int keyring { return NULL; } - if (cert && OPTS_DEBUG(ctx->opts)) + if (keyring && OPTS_DEBUG(ctx->opts)) { - log_dbg_printf("Target cert by origcrt\n"); + log_dbg_printf("Target keyring by origcrt\n"); } } - if (cert) + if (keyring) { ctx->immutable_cert = 1; } } - if (!cert && ctx->origcrt && ctx->opts->key) + if (!keyring && ctx->origcrt && ctx->opts->key) { - cert = cert_new(); - cert->crt = (X509 *) cachemgr_fkcrt_get(ctx->origcrt); + keyring = cert_new(); + keyring->cert = (X509 *) cachemgr_fkcrt_get(ctx->origcrt); - if (cert->crt) + if (keyring->cert) { if (OPTS_DEBUG(ctx->opts)) log_dbg_printf("Certificate cache: HIT\n"); } else { if (OPTS_DEBUG(ctx->opts)) log_dbg_printf("Certificate cache: MISS\n"); - cert->crt = ssl_x509_forge(ctx->opts->cacrt, + keyring->cert = ssl_x509_forge(ctx->opts->cacrt, ctx->opts->cakey, ctx->origcrt, ctx->opts->key, NULL, ctx->opts->crlurl); - cachemgr_fkcrt_set(ctx->origcrt, cert->crt); + cachemgr_fkcrt_set(ctx->origcrt, keyring->cert); } - cert_set_key(cert, ctx->opts->key); - cert_set_chain(cert, ctx->opts->chain); + cert_set_key(keyring, ctx->opts->key); + cert_set_chain(keyring, ctx->opts->chain); ctx->generated_cert = 1; } @@ -306,18 +306,18 @@ void cert_mgr_async_get(struct future* future, struct cert_mgr* mgr, int keyring } if ((WANT_CONNECT_LOG(ctx) || ctx->opts->certgen_writeall) && - cert && cert->crt) + keyring && keyring->cert) { - ctx->usedcrtfpr = ssl_x509_fingerprint(cert->crt, 0); + ctx->usedcrtfpr = ssl_x509_fingerprint(keyring->cert, 0); if (!ctx->usedcrtfpr) ctx->enomem = 1; } - return cert; + return keyring; } -void cert_manager_free(cert_t * cert) +void cert_manager_free(cert_t * keyring) { - cert_free(cert); + cert_free(keyring); return; } #endif diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index c6f3565..e254460 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -1,3 +1,9 @@ +/* + * SSL stream implementation. + * Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise. + * Author: zhengchao@iie.ac.cn + * Create: 2018-8-17 + */ #include #include #include @@ -26,231 +32,401 @@ #include #include #include -#include +#include #include #include #include -#define SSL_EX_DATA_IDX_SSLMGR 0 -#define MAX_NET_RETRIES 50 -struct ssl_mgr -{ - int sslcomp; - int no_ssl2; - int no_ssl3; - int no_tls10; - int no_tls11; - int no_tls12; - CONST_SSL_METHOD *(*sslmethod)(void); //Parameter of SSL_CTX_new - int sslversion; - char ssl_session_context[8]; - int cache_slot_num; - int sess_expire_seconds; - struct sess_cache* down_sess_cache; - struct sess_cache* up_sess_cache; - char* default_ciphers[TFE_SYMBOL_MAX]; - DH * dh; - char * ecdhcurve; - char * crlurl; - uint8_t SSL_MODE_RELEASE_BUFFERS; - void *logger; -}; -struct ssl_stream -{ - enum tfe_conn_dir dir; - SSL * ssl; - struct ssl_mgr* ref_mgr; - char* sni; -}; -struct ssl_chello -{ - //client hello - int version; - char* sni; - char* cipher_suites; + +#define SSL_EX_DATA_IDX_SSLMGR 0 +#define MAX_NET_RETRIES 50 + + +struct ssl_mgr { + int sslcomp; + int no_ssl2; + int no_ssl3; + int no_tls10; + int no_tls11; + int no_tls12; + CONST_SSL_METHOD * (*sslmethod) (void); //Parameter of SSL_CTX_new + int sslversion; + char ssl_session_context[8]; + int cache_slot_num; + int sess_expire_seconds; + struct sess_cache * down_sess_cache; + struct sess_cache * up_sess_cache; + char default_ciphers[TFE_STRING_MAX]; + DH * dh; + char * ecdhcurve; + char * crlurl; + uint8_t SSL_MODE_RELEASE_BUFFERS; + void * logger; + char trust_CA_file[TFE_STRING_MAX]; + char trust_CA_dir[TFE_STRING_MAX]; + X509_STORE* trust_CA_store; + struct key_keeper * keeper_of_keys; }; -struct peek_client_hello_ctx +struct __ssl_stream_debug { - unsigned char sni_peek_retries; /* max 64 SNI parse retries */ - struct event* ev; - struct event_base* evbase; + evutil_socket_t fd; }; -struct ssl_connect_origin_ctx -{ - struct bufferevent *bev; - struct ssl_stream* ssl; +struct ssl_stream { + enum tfe_conn_dir dir; + SSL * ssl; + struct ssl_mgr * mgr; + union + { + struct ssl_chello * client_hello; //dir=upstream, a little weird, which send by downstream client. + struct keyring* keyring; //dir=downstream. + }; + struct __ssl_stream_debug _do_not_use; +}; + + +struct ssl_chello { + //client hello + int version; + char * sni; + char * cipher_suites; +}; + + +struct peek_client_hello_ctx { + struct ssl_chello chello; + unsigned char sni_peek_retries; /* max 64 SNI parse retries */ + struct event * ev; + struct event_base * evbase; + void * logger; +}; + + +struct ssl_connect_origin_ctx { + struct bufferevent * bev; + struct ssl_stream * s_stream; + struct ssl_mgr * mgr; struct sockaddr addr; - socklen_t addrlen; + socklen_t addrlen; + void * logger; + + evutil_socket_t fd_upstream; + evutil_socket_t fd_downstream; + struct event_base * evbase; + struct future * f_peek_chello; }; + + +struct ask_keyring_ctx { + int keyring_id; + struct ssl_stream * origin_ssl; + X509 * origin_crt; + int is_origin_crt_vaild; + struct ssl_mgr * ssl_mgr; + evutil_socket_t fd_downstream; + struct event_base * evbase; + + struct future * f_query_cert; + struct bufferevent * bev_down; + struct ssl_stream* downstream; +}; + + /* * SSL shutdown context. */ struct ssl_shutdown_ctx { - struct ssl_stream* s_stream; - struct event_base *evbase; - struct event *ev; - unsigned int retries; + struct ssl_stream * s_stream; + struct event_base * evbase; + struct event * ev; + unsigned int retries; }; -static int parse_ssl_version(const char * version_str) + +struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir, struct ssl_chello * client_hello, struct keyring* crt) { - int sslversion=-1; + struct sockaddr addr; + socklen_t addrlen; + int ret=0; + struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1); + s_stream->dir=dir; + s_stream->mgr=mgr; + s_stream->_do_not_use.fd=fd; + ret = getpeername(fd , &addr, &addrlen); + assert(ret == 0); + switch (dir) + { + case CONN_DIR_DOWNSTREAM: + s_stream->ssl= downstream_ssl_create(mgr, crt); + s_stream->keyring=crt; + break; + case CONN_DIR_UPSTREAM: + s_stream->ssl= upstream_ssl_create(mgr, client_hello, fd); + s_stream->client_hello=client_hello; + break; + default: + assert(0); + } + return s_stream; +} + + +static void ssl_stream_free(struct ssl_stream * s_stream) +{ + SSL_free(s_stream->ssl); + s_stream->ssl = NULL; + switch (s_stream->dir) + { + case CONN_DIR_DOWNSTREAM: + if(s_stream->keyring!=NULL) + { + key_keeper_free_keyring(s_stream->keyring); + s_stream->keyring=NULL; + } + break; + case CONN_DIR_UPSTREAM: + if(s_stream->client_hello!=NULL) + { + ssl_free_chello(s_stream->client_hello); + s_stream->client_hello=NULL; + } + break; + default: + assert(0); + } + + s_stream->mgr = NULL; + free(s_stream); + return; +} + + +static int sslver_str2num(const char * version_str) +{ + int sslversion = -1; + assert(OPENSSL_VERSION_NUMBER >= 0x10100000L); + /* * Support for SSLv2 and the corresponding SSLv2_method(), * SSLv2_server_method() and SSLv2_client_method() functions were * removed in OpenSSL 1.1.0. */ - if (!strcmp(version_str, "ssl3")) + if (!strcmp(version_str, "ssl3")) { - sslversion = SSL3_VERSION; + sslversion = SSL3_VERSION; } else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1")) { - sslversion = TLS1_VERSION; + sslversion = TLS1_VERSION; } - else if (!strcmp(version_str, "tls11")) + else if (!strcmp(version_str, "tls11")) { - sslversion = TLS1_1_VERSION; - } - else if (!strcmp(version_str, "tls12")) + sslversion = TLS1_1_VERSION; + } + else if (!strcmp(version_str, "tls12")) { - sslversion = TLS1_2_VERSION; - } + sslversion = TLS1_2_VERSION; + } else { - sslversion =-1; + sslversion = -1; } + return sslversion; } -void ssl_manager_destroy(struct ssl_mgr* mgr) + + +void ssl_manager_destroy(struct ssl_mgr * mgr) { + if(mgr->keeper_of_keys!=NULL) + { + key_keeper_destroy(mgr->keeper_of_keys); + } + if(mgr->trust_CA_store!=NULL) + { + X509_STORE_free(mgr->trust_CA_store); + mgr->trust_CA_store=NULL; + } free(mgr); } -struct ssl_mgr* ssl_manager_init(const char* ini_profile, const char* section, void* logger) -{ - struct ssl_mgr* mgr=ALLOC(struct ssl_mgr, 1); - int ret=0,value=0; - char version_str[TFE_SYMBOL_MAX]; - mgr->logger=logger; - MESA_load_profile_string_def(ini_profile, section, "ssl_version", version_str, sizeof(version_str),"tls12"); - mgr->sslversion=parse_ssl_version(version_str); - if(mgr->sslversion<0) - { - TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s",version_str); + +struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, void * logger) { + struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1); + int ret = 0, value = 0; + char version_str[TFE_SYMBOL_MAX]; + mgr->logger = logger; + MESA_load_profile_string_def(ini_profile, section, "ssl_version", version_str, sizeof(version_str), "tls12"); + mgr->sslversion = sslver_str2num(version_str); + + if (mgr->sslversion < 0) { + TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str); goto error_out; } - //tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0. - mgr->sslmethod=TLS_method; - MESA_load_profile_int_def(ini_profile, section, "ssl_compression", &(mgr->sslcomp), 1); - MESA_load_profile_int_def(ini_profile, section, "no_ssl2", &(mgr->no_ssl2), 1); - MESA_load_profile_int_def(ini_profile, section, "no_ssl3", &(mgr->no_ssl3), 1); - MESA_load_profile_int_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 1); - MESA_load_profile_int_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0); - MESA_load_profile_int_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0); - MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", &(mgr->cache_slot_num), 4*1024*1024); - MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", &(mgr->sess_expire_seconds), 30*60); - - mgr->up_sess_cache=ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds,CONN_DIR_UPSTREAM); - mgr->down_sess_cache=ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds,CONN_DIR_DOWNSTREAM); - memcpy(mgr->ssl_session_context,"mesa-tfe",sizeof(mgr->ssl_session_context)); + + //tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0. + mgr->sslmethod = TLS_method; + MESA_load_profile_int_def(ini_profile, section, "ssl_compression", & (mgr->sslcomp), 1); + MESA_load_profile_int_def(ini_profile, section, "no_ssl2", & (mgr->no_ssl2), 1); + MESA_load_profile_int_def(ini_profile, section, "no_ssl3", & (mgr->no_ssl3), 1); + MESA_load_profile_int_def(ini_profile, section, "no_tls10", & (mgr->no_tls10), 1); + MESA_load_profile_int_def(ini_profile, section, "no_tls11", & (mgr->no_tls11), 0); + MESA_load_profile_int_def(ini_profile, section, "no_tls12", & (mgr->no_tls12), 0); + MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", & (mgr->cache_slot_num), + 4 * 1024 * 1024); + MESA_load_profile_int_def(ini_profile, section, "session_cache_slot_num", & (mgr->sess_expire_seconds), 30 * 60); + + mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); + mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); + + + + mgr->keeper_of_keys = key_keeper_init(ini_profile, section, logger); + + if (mgr->keeper_of_keys == NULL) + { + TFE_LOG_ERROR(logger, "Certificate Manager initiate failed."); + goto error_out; + } + mgr->trust_CA_store=X509_STORE_new(); + if(mgr->trust_CA_store==NULL) + { + TFE_LOG_ERROR(logger,"Failed at creating X509_STORE"); + goto error_out; + } + ret=X509_STORE_set_default_paths(mgr->trust_CA_store); + if(ret==0) + { + TFE_LOG_ERROR(logger,"Failed at setting default paths for X509_STORE"); + goto error_out; + } + MESA_load_profile_string_def(ini_profile, section, "trust_CA_file", mgr->trust_CA_file, sizeof(mgr->trust_CA_file), ""); + MESA_load_profile_string_def(ini_profile, section, "trust_CA_dir", mgr->trust_CA_dir, sizeof(mgr->trust_CA_dir), ""); + + ret=X509_STORE_load_locations(mgr->trust_CA_store,strlen(mgr->trust_CA_file)>0? mgr->trust_CA_file:NULL + ,strlen(mgr->trust_CA_dir)>0? mgr->trust_CA_dir:NULL); + if(ret==0) + { + TFE_LOG_ERROR(logger,"Failed at setting load locations for X509_STORE"); + goto error_out; + } + memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context)); return mgr; + error_out: + ssl_manager_destroy(mgr); return NULL; } - - -void peek_client_hello_ctx_free(void* ctx) +int ssl_conn_verify_cert(X509_STORE *store, const SSL * ssl) { - struct peek_client_hello_ctx * _ctx=(struct peek_client_hello_ctx *)ctx; + int ret=0; + STACK_OF(X509) * cert_chain = SSL_get_peer_cert_chain(ssl); + if(cert_chain==NULL) + { + // The peer certificate chain is not necessarily available after reusing a session, in which case a NULL pointer is returned. + return 1; + } + X509_STORE_CTX *ctx=X509_STORE_CTX_new(); + X509 * cert = sk_X509_value(cert_chain, 0); + ret=X509_STORE_CTX_init(ctx, store, cert, cert_chain); + assert(ret==1); + + //If a complete chain can be built and validated this function returns 1, otherwise it return zero or negtive code. + ret=X509_verify_cert(ctx); + X509_STORE_CTX_free(ctx); + return (ret==1); +} + + +void peek_client_hello_ctx_free(void * ctx) +{ + struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)ctx; event_free(_ctx->ev); - _ctx->ev = NULL; - free(_ctx->sni); - _ctx->sni=NULL; + _ctx->ev = NULL; + free(_ctx->chello.sni); + _ctx->chello.sni = NULL; + free(_ctx->chello.cipher_suites); + _ctx->chello.cipher_suites = NULL; free(_ctx); return; } -struct ssl_chello* ssl_peek_result_release_chello(future_result_t* result) + + +struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) { - struct ssl_chello* p=(struct ssl_chello* )result, *copy=NULL; - copy=ALLOC(struct ssl_chello*,1); - if(p!=NULL) - { - copy->sni=tfe_strdup(p->sni); - copy->cipher_suites=tfe_strdup(p->cipher_suites); - copy->version=p->version; + struct ssl_chello * p = (struct ssl_chello *)result, *copy = NULL; + copy = ALLOC(struct ssl_chello, 1); + + if (p != NULL) { + copy->sni = tfe_strdup(p->sni); + copy->cipher_suites = tfe_strdup(p->cipher_suites); + copy->version = p->version; } + return copy; -} -void ssl_free_chello(struct ssl_chello* p) +} void ssl_free_chello(struct ssl_chello * p) { - if(p==NULL) - { + if (p == NULL) { return; } + free(p->sni); - p->sni=NULL; + p->sni = NULL; free(p->cipher_suites); - p->cipher_suites=NULL; - p->cipher_suites=NULL; + p->cipher_suites = NULL; free(p); return; } + static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) { - struct promise* promise=(struct promise*)arg; - struct peek_client_hello_ctx* ctx= (struct peek_client_hello_ctx*)promise_get_ctx(promise); - struct ssl_chello *result=NULL; - const char* reason_too_many_retries="too many tries"; - const char* reason_see_no_client_hello="see no client hello"; - const char* reason=NULL; - memset(&result, 0, sizeof(result)); - char* sni=NULL; - unsigned char buf[1024]; - ssize_t n=0; - const unsigned char * chello=NULL; - int rv=0; + struct promise * promise = (struct promise *)arg; + //use promise_get_ctx instead of promise_dettach_ctx for try more times. + struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *)promise_get_ctx(promise); + const char * reason_too_many_retries = "too many tries"; + const char * reason_see_no_client_hello = "see no client hello"; + const char * reason = NULL; + char * sni = NULL; + unsigned char buf[1024]; + ssize_t n = 0; + const unsigned char * chello = NULL; + int rv = 0; - n = recv(fd, buf, sizeof(buf), MSG_PEEK); - if (n == -1) - { - TFE_LOG_ERROR("Error peeking on fd, aborting connection\n"); + n = recv(fd, buf, sizeof(buf), MSG_PEEK); + + if (n == -1) { + TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection\n"); goto failed; } - - if (n == 0) - { + + if (n == 0) { goto failed; } - result=ssl_get_peek_result(NULL); + //todo: parse version and cipher suites. //or we should use sni proxy instead? https://github.com/dlundquist/sniproxy/blob/master/src/tls.c - rv = ssl_tls_clienthello_parse(buf, n, 0, &chello, &result->sni); - if(rv==0) - { + rv = ssl_tls_clienthello_parse(buf, n, 0, &chello, & (ctx->chello.sni)); + + if (rv == 0) { promise_dettach_ctx(promise); - promise_success(promise, result); - ssl_free_chello(result); + promise_success(promise, & (ctx->chello)); peek_client_hello_ctx_free(ctx); } - else - { - if(!chello) - { - TFE_LOG_ERROR("Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); - reason="see no client hello"; - goto failed; - } - - if(ctx->sni_peek_retries++ > MAX_NET_RETRIES) - { - TFE_LOG_ERROR("Peek failed due to too many retries\n"); - reason="too many peek retries"; + else { + if (!chello) { + TFE_LOG_ERROR(ctx->logger, "Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); + reason = "see no client hello"; goto failed; } + + if (ctx->sni_peek_retries++ > MAX_NET_RETRIES) { + TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n"); + reason = "too many peek retries"; + goto failed; + } + /* ssl_tls_clienthello_parse indicates that we * should retry later when we have more data, and we * haven't reached the maximum retry count yet. @@ -258,121 +434,142 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) * order to prevent busy looping over the read event. * Because we only peeked at the pending bytes and * never actually read them, fd is still ready for - * reading now. We use 25 * 0.2 s = 5 s timeout. */ - struct timeval retry_delay = {0, 100}; + * reading now. We use 25 * 0.2 s = 5 s timeout. */ + struct timeval retry_delay = { + 0, 100 + }; + + event_free(ctx->ev); - ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); - assert(ctx->ev!=NULL); + ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); + assert(ctx->ev != NULL); event_add(ctx->ev, &retry_delay); } + return; + failed: promise_dettach_ctx(promise); - promise_failed(promise,FUTURE_ERROR_EXCEPTION,reason); + promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason); peek_client_hello_ctx_free(ctx); - ssl_free_chello(result); return; } -void ssl_async_peek_client_hello(struct future* future, evutil_socket_t fd, struct event_base *evbase) +static void ssl_async_peek_client_hello(struct future * future, evutil_socket_t fd, struct event_base * evbase, + void * logger) { - struct event * ev=NULL; - struct promise* p=future_to_promise(future); - - struct peek_client_hello_ctx* ctx=ALLOC(struct peek_client_hello_ctx, 1); - ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); + struct event * ev = NULL; + struct promise * p = future_to_promise(future); + struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1); + ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); + ctx->logger = logger; event_add(evbase, NULL); promise_set_ctx(p, ctx, peek_client_hello_ctx_free); return; } + + /* * Create new SSL context for outgoing connections to the original destination. * If hostname sni is provided, use it for Server Name Indication. */ - -static SSL* upstream_ssl_create(struct ssl_mgr* mgr, const struct ssl_chello* chello, struct sockaddr* addr, socklen_t addrlen) +static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, evutil_socket_t fd) { - SSL_CTX* sslctx=NULL; - SSL* ssl=NULL; - SSL_SESSION* sess=NULL; - + SSL_CTX * sslctx = NULL; + SSL * ssl = NULL; + SSL_SESSION * sess = NULL; - sslctx = SSL_CTX_new(mgr->sslmethod()); + + sslctx = SSL_CTX_new(mgr->sslmethod()); sslctx_set_opts(sslctx, mgr); -#if OPENSSL_VERSION_NUMBER >= 0x10100000L +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + if (mgr->sslversion) { if (SSL_CTX_set_min_proto_version(sslctx, chello->version) == 0 || - SSL_CTX_set_max_proto_version(sslctx, chello->version) == 0) { + SSL_CTX_set_max_proto_version(sslctx, chello->version) == 0) { SSL_CTX_free(sslctx); return NULL; } } + #endif /* OPENSSL_VERSION_NUMBER >= 0x10100000L */ SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); - ssl = SSL_new(sslctx); - SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ - if (!ssl) - { + ssl = SSL_new(sslctx); + SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ + + if (!ssl) { return NULL; } + #ifndef OPENSSL_NO_TLSEXT - if (chello->sni) - { + + if (chello->sni) { SSL_set_tlsext_host_name(ssl, sni); } + #endif /* !OPENSSL_NO_TLSEXT */ #ifdef SSL_MODE_RELEASE_BUFFERS + /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); #endif /* SSL_MODE_RELEASE_BUFFERS */ - //Todo: Refactor below session resumption. + struct sockaddr addr; + socklen_t addrlen; + int ret=0; + ret = getpeername(fd , &addr, &addrlen); + assert(ret==0); /* session resuming based on remote endpoint address and port */ - sess = up_session_get(mgr->up_sess_cache, addr, - addrlen, chello->sni); /* new sess inst */ - if (sess) + sess = up_session_get(mgr->up_sess_cache, &addr, addrlen, chello->sni); /* new sess insert */ + if (sess) { - SSL_set_session(ssl, sess); /* increments sess refcount */ + SSL_set_session(ssl, sess); /* increments sess refcount */ SSL_SESSION_free(sess); } - return ssl; } -void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx* ctx) + +void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx) { - if(ctx->ssl!=NULL) - { - ssl_stream_shutdown(ctx->bev,ctx->ssl); - free(ctx->ssl); + if (ctx->s_stream != NULL) { + ssl_stream_free(ctx->s_stream); } - if(ctx->bev!=NULL) - { - bufferevent_free(ctx->bev); - ctx->bev=NULL; + + if (ctx->bev != NULL) { + bufferevent_free(ctx->bev); + ctx->bev = NULL; } + + if (ctx->f_peek_chello != NULL) { + future_destroy(ctx->f_peek_chello); + ctx->f_peek_chello = NULL; + } + free(ctx); return; } -struct ssl_stream* ssl_conn_origin_result_release_stream(future_result_t* result) -{ - struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx* )result; - struct ssl_stream* ret=ctx->ssl; - ctx->ssl=NULL; //giveup ownership + + +struct ssl_stream * ssl_conn_origin_result_release_stream(future_result_t * result) { + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)result; + struct ssl_stream * ret = ctx->s_stream; + ctx->s_stream = NULL; //giveup ownership return ret; } -struct bufferevent* ssl_conn_origin_result_release_bev(future_result_t* result) -{ - struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx* )result; - struct bufferevent * ret=ctx->bev; - ctx->bev=NULL; //giveup ownership + + +struct bufferevent * ssl_conn_origin_result_release_bev(future_result_t * result) { + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)result; + struct bufferevent * ret = ctx->bev; + ctx->bev = NULL; //giveup ownership return ret; } @@ -383,54 +580,83 @@ struct bufferevent* ssl_conn_origin_result_release_bev(future_result_t* result) */ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg) { - struct promise* promise=(struct promise*)arg; - struct ssl_connect_origin_ctx* ctx=(struct ssl_connect_origin_ctx*)promise_get_ctx(promise); - SSL_SESSION * to_cached=NULL; - struct tfe_stream_private* _stream = (struct tfe_stream_private*)arg; - - if (events & BEV_EVENT_ERROR) - { - promise_failed(promise, FUTURE_ERROR_EXCEPTION, "ssl connect failed"); - } - else - { - if (events & BEV_EVENT_CONNECTED) - { - bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL);//leave a clean bev for on_success - to_cached=SSL_get0_session(ctx->ssl->ssl); - up_session_set(ctx->ssl->ref_mgr->up_sess_cache, &(ctx->addr), ctx->addrlen, ctx->ssl->sni, to_cached); - promise_dettach_ctx(promise); - promise_success(promise, ctx); - ctx->bev=NULL; - } - else - { + struct promise * promise = (struct promise *)arg; + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)promise_dettach_ctx(promise); + + struct ssl_stream * s_stream = ctx->s_stream; + SSL_SESSION * ssl_sess = NULL; + + if (events & BEV_EVENT_ERROR) { + promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to orignal server failed."); + } + else { + if (events & BEV_EVENT_CONNECTED) { + bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success + ssl_sess = SSL_get0_session(s_stream->ssl); + up_session_set(s_stream->mgr->up_sess_cache, & (ctx->addr), ctx->addrlen, s_stream->client_hello.sni, + ssl_sess); + promise_success(promise, ctx); + } + else { assert(0); } } + ssl_connect_origin_ctx_free(ctx); return } -void ssl_async_upstream_create(struct future* f, struct ssl_mgr* mgr, const struct ssl_chello* chello, evutil_socket_t fd, struct event_base *evbase) -{ - struct promise* p=future_to_promise(future); - struct ssl_connect_origin_ctx* ctx=ALLOC(struct ssl_connect_origin_ctx, 1); - int ret=0; - struct sockaddr addr; - socklen_t addrlen; - ret=getpeername(fd, &(ctx->addr), &(ctx->addrlen)); - assert(ret==0); - ctx->ssl=ALLOC(struct ssl_stream,1); - ctx->ssl->ref_mgr=mgr; - ctx->ssl->dir=CONN_DIR_UPSTREAM; - ctx->ssl->sni=tfe_strdup(chello->sni); - ctx->ssl->ssl=upstream_ssl_create(mgr, chello, &(ctx->addr), ctx->addrlen); - ctx->bev = bufferevent_openssl_socket_new(evbase, fd, ctx->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); - promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free); + +static void peek_chello_on_succ(future_result_t * result, void * user) +{ + struct promise* p= (struct promise*)user; + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); + + struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. + ctx->s_stream = ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL); + ctx->bev = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_upstream, ctx->s_stream, + BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1); - bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, p); - bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); + bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, ctx); + bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only + future_destroy(ctx->f_peek_chello); + ctx->f_peek_chello = NULL; + return; +} + + +static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user) +{ + struct promise* p= (struct promise*)user; + struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); + + promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream."); + ssl_connect_origin_ctx_free(ctx); + return; +} + + +extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, + evutil_socket_t fd_downstream, struct event_base * evbase) +{ + struct promise * p = future_to_promise(f); + struct ssl_connect_origin_ctx * ctx = ALLOC(struct ssl_connect_origin_ctx, 1); + int ret = 0; + + struct sockaddr addr; + socklen_t addrlen; + + ret = getpeername(fd_downstream, & (ctx->addr), & (ctx->addrlen)); + assert(ret == 0); + ctx->fd_downstream = fd_downstream; + ctx->fd_upstream = fd_upstream; + ctx->evbase = evbase; + ctx->mgr = mgr; + promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free); + + ctx->f_peek_chello = future_create(peek_chello_on_succ, peek_chello_on_fail, p); + ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); + return; } @@ -441,117 +667,142 @@ void ssl_async_upstream_create(struct future* f, struct ssl_mgr* mgr, const stru * the refcount decrementing. In other words, return 0 if we did not * keep a pointer to the object (which we never do here). */ - -static int -ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) +static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) { - struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); + struct ssl_mgr * mgr = (struct ssl_mgr *) + SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); #ifdef HAVE_SSLV2 + /* Session resumption seems to fail for SSLv2 with protocol * parsing errors, so we disable caching for SSLv2. */ if (SSL_version(ssl) == SSL2_VERSION) { return 0; } + #endif /* HAVE_SSLV2 */ - if (sess) - { + + if (sess) { down_session_set(mgr->down_sess_cache, sess); } + return 0; } + /* * Called by OpenSSL when a src SSL session should be removed. * OpenSSL calls SSL_SESSION_free() after calling the callback; * we do not need to free the reference here. */ -static void -ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) +static void ossl_sessremove_cb(UNUSED SSL_CTX * sslctx, SSL_SESSION * sess) { - struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); - assert(mgr!=NULL); - if (sess) - { - down_session_del(mgr->down_sess_cache, sess); + struct ssl_mgr * mgr = (struct ssl_mgr *) + SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); + assert(mgr != NULL); + + if (sess) { + down_session_del(mgr->down_sess_cache, sess) ; } + return; } + /* * Called by OpenSSL when a src SSL session is requested by the client. - OPENSSL_VERSION_NUMBER >= 0x10100000L required. + OPENSSL_VERSION_NUMBER >= 0x10100000L required. */ -static SSL_SESSION * -ossl_sessget_cb(UNUSED SSL *ssl, const unsigned char *id, int idlen, int *copy) +static SSL_SESSION * ossl_sessget_cb(UNUSED SSL * ssl, const unsigned char * id, int idlen, int * copy) { - struct ssl_mgr* mgr=(struct ssl_mgr*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); + struct ssl_mgr * mgr = (struct ssl_mgr *) + SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); SSL_SESSION * sess; - *copy = 0; /* SSL should not increment reference count of session */ - sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache,id, idlen); + *copy = 0; /* SSL should not increment reference count of session */ + sess = (SSL_SESSION *) + down_session_get(mgr->down_sess_cache, id, idlen); return sess; } + /* * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX. */ -static void -sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr* mgr) +static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr) { SSL_CTX_set_options(sslctx, SSL_OP_ALL); + #ifdef SSL_OP_TLS_ROLLBACK_BUG SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG); #endif /* SSL_OP_TLS_ROLLBACK_BUG */ + #ifdef SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); #endif /* SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION */ + #ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); #endif /* SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS */ + #ifdef SSL_OP_NO_TICKET SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET); #endif /* SSL_OP_NO_TICKET */ #ifdef SSL_OP_NO_SSLv2 + #ifdef HAVE_SSLV2 + if (mgr->no_ssl2) { #endif /* HAVE_SSLV2 */ - SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); + + SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); + #ifdef HAVE_SSLV2 } + #endif /* HAVE_SSLV2 */ + #endif /* !SSL_OP_NO_SSLv2 */ + #ifdef HAVE_SSLV3 - if (mgr->no_ssl3) - { + + if (mgr->no_ssl3) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); } + #endif /* HAVE_SSLV3 */ + #ifdef HAVE_TLSV10 - if (mgr->no_tls10) - { + + if (mgr->no_tls10) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1); } + #endif /* HAVE_TLSV10 */ + #ifdef HAVE_TLSV11 - if (mgr->no_tls11) - { + + if (mgr->no_tls11) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1); } + #endif /* HAVE_TLSV11 */ + #ifdef HAVE_TLSV12 - if (mgr->no_tls12) - { + + if (mgr->no_tls12) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2); } + #endif /* HAVE_TLSV12 */ #ifdef SSL_OP_NO_COMPRESSION - if (!mgr->sslcomp) - { + + if (!mgr->sslcomp) { SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION); } + #endif /* SSL_OP_NO_COMPRESSION */ SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); @@ -560,108 +811,183 @@ sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr* mgr) /* * Create and set up a new SSL_CTX instance for terminating SSL. - * Set up all the necessary callbacks, the cert, the cert chain and key. + * Set up all the necessary callbacks, the keyring, the keyring chain and key. */ -static SSL_CTX * down_sslctx_create(struct ssl_mgr* mgr, struct cert* crt) +static SSL* downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt) { - SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); - if (!sslctx) - return NULL; + SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); + if (!sslctx) + return NULL; + SSL* ssl=NULL; + int ret=0; + sslctx_set_opts(sslctx, mgr); - sslctx_set_opts(sslctx, mgr); + //TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L + if (mgr->sslversion) { + if (SSL_CTX_set_min_proto_version(sslctx, mgr->sslversion) == 0 || + SSL_CTX_set_max_proto_version(sslctx, mgr->sslversion) == 0) { + SSL_CTX_free(sslctx); + return NULL; + } + } -//TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L - if (mgr->sslversion) { - if (SSL_CTX_set_min_proto_version(sslctx,mgr->sslversion) == 0 || - SSL_CTX_set_max_proto_version(sslctx, mgr->sslversion) == 0) { - SSL_CTX_free(sslctx); - return NULL; - } - } + SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); + SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); + SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); + SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); + SSL_CTX_set_session_id_context(sslctx, (void *) (mgr->ssl_session_context), + sizeof(mgr->ssl_session_context)); - SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); - SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); - SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); - SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | - SSL_SESS_CACHE_NO_INTERNAL); - SSL_CTX_set_session_id_context(sslctx, (void *)(mgr->ssl_session_context), - sizeof(mgr->ssl_session_context)); #ifndef OPENSSL_NO_DH - if (mgr->dh) - { - SSL_CTX_set_tmp_dh(sslctx, mgr->dh); - } - else - { - SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); - } + + if (mgr->dh) { + SSL_CTX_set_tmp_dh(sslctx, mgr->dh); + } + else { + SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); + } + #endif /* !OPENSSL_NO_DH */ + #ifndef OPENSSL_NO_ECDH - if (mgr->ecdhcurve) - { - EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); - SSL_CTX_set_tmp_ecdh(sslctx, ecdh); - EC_KEY_free(ecdh); - } - else - { - EC_KEY * ecdh = ssl_ec_by_name(NULL); - SSL_CTX_set_tmp_ecdh(sslctx, ecdh); - EC_KEY_free(ecdh); - } + + if (mgr->ecdhcurve) { + EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); + SSL_CTX_set_tmp_ecdh(sslctx, ecdh); + EC_KEY_free(ecdh); + } + else { + EC_KEY * ecdh = ssl_ec_by_name(NULL); + SSL_CTX_set_tmp_ecdh(sslctx, ecdh); + EC_KEY_free(ecdh); + } + #endif /* !OPENSSL_NO_ECDH */ - SSL_CTX_use_certificate(sslctx, crt->crt); - SSL_CTX_use_PrivateKey(sslctx, crt->key); - for (int i = 0; i < sk_X509_num(crt->chain); i++) - { - X509 * c = sk_X509_value(crt->chain, i); - ssl_x509_refcount_inc(c); /* next call consumes a reference */ - SSL_CTX_add_extra_chain_cert(sslctx, c); - } - return sslctx; + + SSL_CTX_use_certificate(sslctx, crt->cert); + SSL_CTX_use_PrivateKey(sslctx, crt->key); + + for (int i = 0; i < sk_X509_num(crt->chain); i++) { + X509 * c = sk_X509_value(crt->chain, i); + ssl_x509_refcount_inc(c); /* next call consumes a reference */ + SSL_CTX_add_extra_chain_cert(sslctx, c); + } + ssl=SSL_new(sslctx); + SSL_CTX_free(sslctx); // SSL_new() increments refcount + sslctx=NULL; + ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); + assert(ret == 1); + + if (mgr->SSL_MODE_RELEASE_BUFFERS == 1) + { + /* lower memory footprint for idle connections */ + SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); + } + + return ssl; +} + + +void query_cert_ctx_free(struct ask_keyring_ctx * ctx) +{ + X509_free(ctx->origin_crt); + + if (ctx->f_query_cert != NULL) { + future_destroy(ctx->f_query_cert); + ctx->f_query_cert = NULL; + } + if(ctx->bev_down!=NULL) + { + bufferevent_free(ctx->bev_down); + } + if(ctx->downstream!=NULL) + { + ssl_stream_free(ctx->downstream); + } + return; +} + +struct ssl_stream* ssl_downstream_create_result_release_stream(future_result_t* result) +{ + struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)result; + struct ssl_stream* ret=ctx->downstream; + ctx->downstream=NULL; + return ret; +} +struct bufferevent* ssl_downstream_create_result_release_bev(future_result_t* result) +{ + struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)result; + struct bufferevent* ret=ctx->bev_down; + ctx->bev_down=NULL; + return ret; +} + +void ask_keyring_on_succ(void * result, void * user) +{ + struct promise* p=(struct promise*)user; + struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)promise_dettach_ctx(promise); + + struct ssl_stream * downstream = NULL; + struct keyring* crt=NULL; + + struct ssl_mgr* mgr=ctx->ssl_mgr; + + future_destroy(ctx->f_query_cert); + ctx->f_query_cert=NULL; + + crt=key_keeper_release_cert(result); + + ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, crt); + + ctx->bev_down = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl, + BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); + bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); + + promise_success(p, ctx); + key_keeper_free_keyring(crt); + query_cert_ctx_free(ctx); + return; +} + + +void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user) +{ + struct promise* p=(struct promise*)user; + struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)promise_dettach_ctx(promise); + promise_failed(p, error, what); + query_cert_ctx_free(ctx); + return; } /* - * Create new SSL context for the incoming connection, based on the a designate cert - * Returns NULL when openssl error. + * Create a SSL stream for the incoming connection, based on the upstream. */ -void ssl_async_downstream_create(struct future* f, struct ssl_mgr* mgr, struct ssl_chello* hello, evutil_socket_t fd, struct event_base *evbase) +void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base * evbase) { - struct cert* crt; - struct ssl_stream* p=NULL; - if(crt==NULL) + + assert(upstream->dir == CONN_DIR_UPSTREAM); + + struct ask_keyring_ctx * ctx = ALLOC(struct ask_keyring_ctx, 1); + ctx->keyring_id = keyring_id; + ctx->ssl_mgr = mgr; + ctx->fd_downstream = fd_downstream; + ctx->evbase = evbase; + + if(upstream!=NULL) { - goto error_out; + ctx->origin_ssl = upstream; + ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl); } - p=ALLOC(struct ssl_stream, 1); - p->ref_mgr=mgr; - p->dir=CONN_DIR_DOWNSTREAM; - //could be optimized by store the sslctx. - SSL_CTX * sslctx = down_sslctx_create(mgr,crt); - cert_mgr_free_cert(crt); - if (!sslctx) - { - goto error_out; - } - p->ssl = SSL_new(sslctx); - int ret=SSL_set_ex_data(p->ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); - assert(ret==1); - SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ - if (!p->ssl) - { - goto error_out; - } - if(mgr->SSL_MODE_RELEASE_BUFFERS==1) - { - /* lower memory footprint for idle connections */ - SSL_set_mode(p->ssl, SSL_get_mode(p->ssl) | SSL_MODE_RELEASE_BUFFERS); - } - p->sni=tfe_strdup(hello->sni); - return p; -error_out: - free(p); - return NULL; + + struct promise* p = future_to_promise(f); + promise_set_ctx(p, ctx, query_cert_ctx_free); + ctx->is_origin_crt_vaild=ssl_conn_verify_cert(mgr->trust_CA_store, upstream->ssl); + + ctx->f_query_cert = future_create(ask_keyring_on_succ, ask_keyring_on_fail, p); + //todo add a is_valid_cert flag to keyring manager query API. + key_keeper_async_ask(ctx->f_query_cert, mgr->keeper_of_keys, keyring_id, ctx->origin_crt, ctx->is_origin_crt_vaild, evbase); + return; } @@ -672,48 +998,42 @@ error_out: * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE * may or may not work. */ -static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream* s_stream, struct event_base *evbase) -{ - struct ssl_shutdown_ctx *ctx = ALLOC(struct ssl_shutdown_ctx,1); +static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase) { + struct ssl_shutdown_ctx * ctx = ALLOC(struct ssl_shutdown_ctx, 1); ctx->evbase = evbase; ctx->s_stream = s_stream; ctx->ev = NULL; ctx->retries = 0; return ctx; -} +} -static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx *ctx) +static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx) { free(ctx); } -static void ssl_stream_free(struct ssl_stream* s_stream) -{ - SSL_free(s_stream->ssl); - s_stream->ssl=NULL; - free(s_stream->sni); - s_stream->sni=NULL; - s_stream->ref_mgr=NULL - free(s_stream); - return; -} + + /* * The shutdown socket event handler. This is either * scheduled as a timeout-only event, or as a fd read or * fd write event, depending on whether SSL_shutdown() * indicates it needs read or write on the socket. */ -static void -pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void *arg) +static void pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void * arg) { - struct ssl_shutdown_ctx *ctx = (struct ssl_shutdown_ctx *)arg; - struct timeval retry_delay = {0, 100}; - void *logger=ctx->s_stream->ref_mgr->logger; - short want = 0; - int rv=0, sslerr=0; + struct ssl_shutdown_ctx * ctx = (struct ssl_shutdown_ctx *) arg; + + struct timeval retry_delay = { + 0, 100 + }; + + void * logger = ctx->s_stream->mgr->logger; + short want = 0; + int rv = 0, sslerr = 0; if (ctx->ev) { event_free(ctx->ev); - ctx->ev = NULL; + ctx->ev = NULL; } /* @@ -725,13 +1045,17 @@ pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void *arg) * This is a good collection of recent and relevant documents: * http://bugs.python.org/issue8108 */ - rv = SSL_shutdown(ctx->s_stream->ssl); + rv = SSL_shutdown(ctx->s_stream->ssl); + if (rv == 1) goto complete; + if (rv != -1) { goto retry; } - switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) { + + switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) + { case SSL_ERROR_WANT_READ: want = EV_READ; goto retry; @@ -745,24 +1069,30 @@ pxy_ssl_shutdown_cb(evutil_socket_t fd, UNUSED short what, void *arg) goto complete; default: TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() " - "error %i. Closing fd.\n", sslerr); + "error %i. Closing fd.\n", sslerr); goto complete; } + goto complete; retry: + if (ctx->retries++ >= MAX_NET_RETRIES) { - TFE_LOG_ERROR(logger,"Failed to shutdown SSL connection cleanly: " - "Max retries reached. Closing fd.\n"); + TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " + "Max retries reached. Closing fd.\n"); goto complete; } - ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); + + ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); + if (ctx->ev) { event_add(ctx->ev, &retry_delay); return; } - TFE_LOG_ERROR(logger,"Failed to shutdown SSL connection cleanly: " - "Cannot create event. Closing fd.\n"); + + TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " + "Cannot create event. Closing fd.\n"); + complete: ssl_stream_free(ctx->s_stream); @@ -771,16 +1101,17 @@ complete: } + /* * Cleanly shutdown an SSL session on file descriptor fd using low-level * file descriptor readiness events on event base evbase. * Guarantees that SSL and the corresponding SSL_CTX are freed and the * socket is closed, eventually, or in the case of fatal errors, immediately. */ -void ssl_stream_free_and_close_fd(struct ssl_stream* s_stream, struct event_base *evbase, evutil_socket_t fd) +void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, evutil_socket_t fd) { - struct ssl_shutdown_ctx *sslshutctx=NULL; - sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); + struct ssl_shutdown_ctx * sslshutctx = NULL; + sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); pxy_ssl_shutdown_cb(fd, 0, sslshutctx); return; }