/* * SSL stream implementation. * Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise. * Author: zhengchao@iie.ac.cn * Create: 2018-8-17 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SSL_EX_DATA_IDX_SSLMGR 0 #define MAX_NET_RETRIES 50 /* * Default cipher suite spec. * Use 'openssl ciphers -v spec' to see what ciphers are effectively enabled * by a cipher suite spec with a given version of OpenSSL. */ #define DFLT_CIPHERS "ALL:-aNULL" /* * Default elliptic curve for EC cipher suites. */ #define DFLT_CURVE "prime256v1" enum ssl_stream_stat { SSL_UP_NUM, SSL_UP_ERR, SSL_UP_CLOSED, SSL_DOWN_NUM, SSL_DOWN_ERR, SSL_DOWN_CLOSED, SSL_NO_CHELLO, SSL_NO_SNI, SSL_FAKE_CRT, SSL_STAT_MAX }; struct ssl_mgr { unsigned int sslcomp; unsigned int no_ssl2; unsigned int no_ssl3; unsigned int no_tls10; unsigned int no_tls11; unsigned int no_tls12; CONST_SSL_METHOD * (* sslmethod)(void); //Parameter of SSL_CTX_new int ssl_min_version, ssl_max_version; char ssl_session_context[8]; unsigned int cache_slots; unsigned int sess_expire_seconds; struct sess_cache * down_sess_cache; struct sess_cache * up_sess_cache; char default_ciphers[TFE_STRING_MAX]; DH * dh; char * ecdhcurve; char * crl_url; uint8_t ssl_mode_release_buffers; char trust_CA_file[TFE_STRING_MAX]; char trust_CA_dir[TFE_STRING_MAX]; X509_STORE * trust_CA_store; struct key_keeper * keeper_of_keys; struct event_base * ev_base_gc; struct event * gcev; void * logger; screen_stat_handle_t* fs_handle; long long stat_val[SSL_STAT_MAX]; int fs_id[SSL_STAT_MAX]; }; struct __ssl_stream_debug { evutil_socket_t fd; }; struct ssl_stream { enum tfe_conn_dir dir; SSL * ssl; struct ssl_mgr * mgr; union { struct ssl_chello * client_hello; //dir=upstream, a little weird, which send by downstream client. struct keyring * keyring; //dir=downstream. }; struct __ssl_stream_debug _do_not_use; }; struct peek_client_hello_ctx { struct ssl_chello* chello; unsigned char sni_peek_retries; /* max 64 SNI parse retries */ struct event * ev; struct event_base * evbase; void * logger; }; struct ssl_connect_origin_ctx { struct bufferevent * bev; struct ssl_stream * s_stream; struct ssl_mgr * mgr; struct sockaddr_storage addr; socklen_t addrlen; void * logger; evutil_socket_t fd_upstream; evutil_socket_t fd_downstream; struct event_base * evbase; struct future * f_peek_chello; }; struct ask_keyring_ctx { int keyring_id; struct ssl_stream * origin_ssl; X509 * origin_crt; int is_origin_crt_vaild; struct ssl_mgr * ssl_mgr; evutil_socket_t fd_downstream; struct event_base * evbase; struct future * f_query_cert; struct bufferevent * bev_down; struct ssl_stream * downstream; }; /* * SSL shutdown context. */ struct ssl_shutdown_ctx { struct ssl_stream * s_stream; struct event_base * evbase; struct event * ev; unsigned int retries; }; struct fs_spec { enum ssl_stream_stat id; const char* name; }; void ssl_stat_init(struct ssl_mgr * mgr) { int i=0; const char* spec[SSL_STAT_MAX]= { [SSL_UP_NUM]="ssl_up", [SSL_UP_ERR]="ssl_up_err", [SSL_UP_CLOSED]="ssl_up_close", [SSL_DOWN_NUM]="ssl_down", [SSL_DOWN_ERR]="ssl_down_err", [SSL_DOWN_CLOSED]="ssl_down_close", [SSL_NO_CHELLO]="ssl_no_chello", [SSL_NO_SNI]="ssl_no_sni", [SSL_FAKE_CRT]="ssl_fake_crt"}; for(i=0;ifs_id[i]=FS_register(mgr->fs_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,spec[i]); } return; } static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt); static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, evutil_socket_t fd); static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr); struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) { struct peek_client_hello_ctx* ctx= (struct peek_client_hello_ctx*) result; struct ssl_chello * p = ctx->chello; ctx->chello=NULL; return p; } struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir, struct ssl_chello * client_hello, struct keyring * kyr) { int ret = 0; struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1); s_stream->dir = dir; s_stream->mgr = mgr; s_stream->_do_not_use.fd = fd; assert(ret == 0); switch (dir) { case CONN_DIR_DOWNSTREAM: s_stream->ssl = downstream_ssl_create(mgr, kyr); s_stream->keyring = kyr; break; case CONN_DIR_UPSTREAM: s_stream->ssl = upstream_ssl_create(mgr, client_hello, fd); s_stream->client_hello = client_hello; break; default: assert(0); } return s_stream; } static void ssl_stream_free(struct ssl_stream * s_stream) { SSL_free(s_stream->ssl); s_stream->ssl = NULL; switch (s_stream->dir) { case CONN_DIR_DOWNSTREAM: if (s_stream->keyring != NULL) { key_keeper_free_keyring(s_stream->keyring); s_stream->keyring = NULL; } break; case CONN_DIR_UPSTREAM: if (s_stream->client_hello != NULL) { ssl_chello_free(s_stream->client_hello); s_stream->client_hello = NULL; } break; default: assert(0); } s_stream->mgr = NULL; free(s_stream); return; } static int sslver_str2num(const char * version_str) { int sslversion = -1; assert(OPENSSL_VERSION_NUMBER >= 0x10100000L); /* * Support for SSLv2 and the corresponding SSLv2_method(), * SSLv2_server_method() and SSLv2_client_method() functions were * removed in OpenSSL 1.1.0. */ if (!strcmp(version_str, "ssl3")) { sslversion = SSL3_VERSION; } else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1")) { sslversion = TLS1_VERSION; } else if (!strcmp(version_str, "tls11")) { sslversion = TLS1_1_VERSION; } else if (!strcmp(version_str, "tls12")) { sslversion = TLS1_2_VERSION; } else { sslversion = -1; } return sslversion; } void ssl_manager_destroy(struct ssl_mgr * mgr) { if (mgr->keeper_of_keys != NULL) { key_keeper_destroy(mgr->keeper_of_keys); } if (mgr->trust_CA_store != NULL) { X509_STORE_free(mgr->trust_CA_store); mgr->trust_CA_store = NULL; } if(mgr->down_sess_cache!=NULL) { ssl_sess_cache_destroy(mgr->down_sess_cache); } if(mgr->up_sess_cache!=NULL) { ssl_sess_cache_destroy(mgr->up_sess_cache); } if(mgr->gcev!=NULL) { event_free(mgr->gcev); } free(mgr); } /* * Garbage collection handler. */ static void ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) { struct ssl_mgr *mgr=(struct ssl_mgr *)arg; int i=0; return; for(i=0;ifs_handle, i, 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i]))); } return; } struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, struct event_base * ev_base_gc, void * logger, screen_stat_handle_t * fs) { struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here. struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1); int ret = 0; char version_str[TFE_SYMBOL_MAX]; mgr->logger = logger; mgr->ev_base_gc=ev_base_gc; mgr->fs_handle=fs; MESA_load_profile_string_def(ini_profile, section, "ssl_min_version", version_str, sizeof(version_str), "ssl3"); mgr->ssl_min_version = sslver_str2num(version_str); MESA_load_profile_string_def(ini_profile, section, "ssl_max_version", version_str, sizeof(version_str), "tls12"); mgr->ssl_max_version = sslver_str2num(version_str); if (mgr->ssl_min_version < 0) { TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str); goto error_out; } ret=ssl_init(); if(ret<0) { TFE_LOG_ERROR(logger, "OpenSSL global init failed."); goto error_out; } //tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0. mgr->sslmethod = TLS_method; MESA_load_profile_uint_def(ini_profile, section, "ssl_compression", &(mgr->sslcomp), 1); MESA_load_profile_uint_def(ini_profile, section, "no_ssl2", &(mgr->no_ssl2), 1); MESA_load_profile_uint_def(ini_profile, section, "no_ssl3", &(mgr->no_ssl3), 1); MESA_load_profile_uint_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 1); MESA_load_profile_uint_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0); MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers, sizeof(mgr->default_ciphers), DFLT_CIPHERS); MESA_load_profile_uint_def(ini_profile, section, "session_cache_slots", &(mgr->cache_slots), 4 * 1024 * 1024); MESA_load_profile_uint_def(ini_profile, section, "session_cache_slots", &(mgr->sess_expire_seconds), 30 * 60); mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); mgr->keeper_of_keys = key_keeper_init(ini_profile, section, logger); #if 0 if (mgr->keeper_of_keys == NULL) { TFE_LOG_ERROR(logger, "Certificate Manager initiate failed."); goto error_out; } #endif mgr->trust_CA_store = X509_STORE_new(); if (mgr->trust_CA_store == NULL) { TFE_LOG_ERROR(logger, "Failed at creating X509_STORE"); goto error_out; } ret = X509_STORE_set_default_paths(mgr->trust_CA_store); if (ret == 0) { TFE_LOG_ERROR(logger, "Failed at setting default paths for X509_STORE"); goto error_out; } MESA_load_profile_string_def(ini_profile, section, "trust_CA_file", mgr->trust_CA_file, sizeof(mgr->trust_CA_file), "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"); MESA_load_profile_string_def(ini_profile, section, "trust_CA_dir", mgr->trust_CA_dir, sizeof(mgr->trust_CA_dir), ""); ret = X509_STORE_load_locations(mgr->trust_CA_store, strlen(mgr->trust_CA_file) > 0 ? mgr->trust_CA_file : NULL, strlen(mgr->trust_CA_dir) > 0 ? mgr->trust_CA_dir : NULL); if (ret == 0) { TFE_LOG_ERROR(logger, "Failed at setting load locations for X509_STORE"); goto error_out; } memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context)); //ssl_stat_init(mgr); mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr); if (!mgr->gcev) { goto error_out; } evtimer_add(mgr->gcev, &gc_delay); return mgr; error_out: ssl_manager_destroy(mgr); return NULL; } int ssl_conn_verify_cert(X509_STORE * store, const SSL * ssl) { int ret = 0; STACK_OF(X509) * cert_chain = SSL_get_peer_cert_chain(ssl); if (cert_chain == NULL) { // The peer certificate chain is not necessarily available after reusing a session, in which case a NULL pointer is returned. return 1; } X509_STORE_CTX * ctx = X509_STORE_CTX_new(); X509 * cert = sk_X509_value(cert_chain, 0); ret = X509_STORE_CTX_init(ctx, store, cert, cert_chain); assert(ret == 1); //If a complete chain can be built and validated this function returns 1, otherwise it return zero or negtive code. ret = X509_verify_cert(ctx); X509_STORE_CTX_free(ctx); return (ret == 1); } void peek_client_hello_ctx_free(struct peek_client_hello_ctx * _ctx) { event_free(_ctx->ev); _ctx->ev = NULL; if(_ctx->chello!=NULL) { ssl_chello_free(_ctx->chello); _ctx->chello=NULL; } free(_ctx); } void peek_client_hello_ctx_free_cb(void * p) { struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)p; return peek_client_hello_ctx_free(_ctx); } static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) { struct promise * promise = (struct promise *) arg; //use promise_get_ctx instead of promise_dettach_ctx for try more times. struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *) promise_get_ctx(promise); enum chello_parse_result chello_status=CHELLO_PARSE_INVALID_FORMAT; struct ssl_chello* chello=NULL; const char * reason = NULL; unsigned char buf[2048]; ssize_t n = 0; n = recv(fd, buf, sizeof(buf), MSG_PEEK); if (n == -1) { TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection\n"); goto failed; } if (n == 0) { goto failed; } chello=ssl_chello_parse(buf,n, &chello_status); switch(chello_status) { case CHELLO_PARSE_SUCCESS: { promise_dettach_ctx(promise); ctx->chello=chello; promise_success(promise, ctx); peek_client_hello_ctx_free(ctx); break; } case CHELLO_PARSE_NOT_ENOUGH_BUFF: { ssl_chello_free(chello); chello=NULL; if (ctx->sni_peek_retries++ > MAX_NET_RETRIES) { TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n"); reason = "too many peek retries"; goto failed; } /* ssl_tls_clienthello_parse indicates that we * should retry later when we have more data, and we * haven't reached the maximum retry count yet. * Reschedule this event as timeout-only event in * order to prevent busy looping over the read event. * Because we only peeked at the pending bytes and * never actually read them, fd is still ready for * reading now. We use 25 * 0.2 s = 5 s timeout. */ struct timeval retry_delay = {0, 100}; event_free(ctx->ev); ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); assert(ctx->ev != NULL); event_add(ctx->ev, &retry_delay); break; } case CHELLO_PARSE_INVALID_FORMAT: { ssl_chello_free(chello); chello=NULL; TFE_LOG_ERROR(ctx->logger, "Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); reason = "see no client hello"; goto failed; break; } default: assert(0); } return; failed: promise_dettach_ctx(promise); promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason); peek_client_hello_ctx_free(ctx); return; } static void ssl_async_peek_client_hello(struct future * future, evutil_socket_t fd, struct event_base * evbase, void * logger) { struct promise * p = future_to_promise(future); struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1); ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); ctx->logger = logger; event_add(ctx->ev, NULL); promise_set_ctx(p, (void *) ctx, peek_client_hello_ctx_free_cb); return; } /* * Create new SSL context for outgoing connections to the original destination. * If hostname sni is provided, use it for Server Name Indication. */ static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, evutil_socket_t fd) { SSL_CTX * sslctx = NULL; SSL * ssl = NULL; SSL_SESSION * sess = NULL; sslctx = SSL_CTX_new(mgr->sslmethod()); sslctx_set_opts(sslctx, mgr); int ret=0; if(chello->cipher_suites!=NULL) { //SSL_CTX_set_cipher_list() and SSL_set_cipher_list() return 1 if any cipher could be selected and 0 on complete failure. ret=SSL_CTX_set_cipher_list(sslctx, chello->cipher_suites); if(ret==0) { TFE_LOG_ERROR(mgr->logger, "SSL_CTX_set_cipher_list %s failed.", chello->cipher_suites); SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); } } else { ret=SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); } if (mgr->ssl_min_version) { if (SSL_CTX_set_min_proto_version(sslctx, MAX(chello->min_version.ossl_format, mgr->ssl_min_version)) == 0 || SSL_CTX_set_max_proto_version(sslctx, MIN(chello->max_version.ossl_format, mgr->ssl_max_version)) == 0) { SSL_CTX_free(sslctx); return NULL; } } SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); ssl = SSL_new(sslctx); SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ if (!ssl) { return NULL; } if (chello->sni) { SSL_set_tlsext_host_name(ssl, chello->sni); } /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); struct sockaddr_storage addr; socklen_t addrlen = sizeof(struct sockaddr_storage); ret = getpeername(fd, (struct sockaddr *) (&addr), &addrlen); assert(ret == 0); /* session resuming based on remote endpoint address and port */ sess = up_session_get(mgr->up_sess_cache, (struct sockaddr *) &addr, addrlen, chello->sni); /* new sess insert */ if (sess) { SSL_set_session(ssl, sess); /* increments sess refcount */ SSL_SESSION_free(sess); } return ssl; } void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx) { if (ctx->s_stream != NULL) { ssl_stream_free(ctx->s_stream); } if (ctx->bev != NULL) { bufferevent_free(ctx->bev); ctx->bev = NULL; } if (ctx->f_peek_chello != NULL) { future_destroy(ctx->f_peek_chello); ctx->f_peek_chello = NULL; } free(ctx); return; } void ssl_connect_origin_ctx_free(void *p) { struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)p; ssl_connect_origin_ctx_free(ctx); } struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result) { struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) result; struct ssl_stream * ret = ctx->s_stream; ctx->s_stream = NULL; //giveup ownership return ret; } struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result) { struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) result; struct bufferevent * ret = ctx->bev; ctx->bev = NULL; //giveup ownership return ret; } void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, void* logger) { unsigned long sslerr=0; const char* dir_str=(dir==CONN_DIR_DOWNSTREAM?"downstream":"upstream"); /* Can happen for socket errs, ssl errs; * may happen for unclean ssl socket shutdowns. */ sslerr = bufferevent_get_openssl_error(bev); if (!errno && !sslerr) { /* We have disabled notification for unclean shutdowns * so this should not happen; log a warning. */ TFE_LOG_ERROR(logger,"Warning: Spurious error from " "bufferevent (errno=0,sslerr=0)\n"); } else if (ERR_GET_REASON(sslerr) == SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE) { /* these can happen due to client cert auth, * only log error if debugging is activated */ TFE_LOG_ERROR(logger,"Handshake Error from %s bufferevent: " "%i:%s %lu:%i:%s:%i:%s:%i:%s\n", dir_str, errno, errno ? strerror(errno) : "-", sslerr, ERR_GET_REASON(sslerr), sslerr ? ERR_reason_error_string(sslerr) : "-", ERR_GET_LIB(sslerr), sslerr ? ERR_lib_error_string(sslerr) : "-", ERR_GET_FUNC(sslerr), sslerr ? ERR_func_error_string(sslerr) : "-"); while ((sslerr = bufferevent_get_openssl_error(bev))) { TFE_LOG_ERROR(logger,"Additional SSL error: " "%lu:%i:%s:%i:%s:%i:%s\n", sslerr, ERR_GET_REASON(sslerr), ERR_reason_error_string(sslerr), ERR_GET_LIB(sslerr), ERR_lib_error_string(sslerr), ERR_GET_FUNC(sslerr), ERR_func_error_string(sslerr)); } } else { /* real errors */ TFE_LOG_ERROR(logger,"Error from %s bufferevent: " "%i:%s %lu:%i:%s:%i:%s:%i:%s\n", dir_str, errno, errno ? strerror(errno) : "-", sslerr, ERR_GET_REASON(sslerr), sslerr ? ERR_reason_error_string(sslerr) : "-", ERR_GET_LIB(sslerr), sslerr ? ERR_lib_error_string(sslerr) : "-", ERR_GET_FUNC(sslerr), sslerr ? ERR_func_error_string(sslerr) : "-"); while ((sslerr = bufferevent_get_openssl_error(bev))) { TFE_LOG_ERROR(logger,"Additional SSL error: " "%lu:%i:%s:%i:%s:%i:%s\n", sslerr, ERR_GET_REASON(sslerr), ERR_reason_error_string(sslerr), ERR_GET_LIB(sslerr), ERR_lib_error_string(sslerr), ERR_GET_FUNC(sslerr), ERR_func_error_string(sslerr)); } } } /* * Callback for meta events on the up- and downstream connection bufferevents. * Called when EOF has been reached, a connection has been made, and on errors. */ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg) { struct promise * promise = (struct promise *) arg; struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(promise); struct ssl_stream * s_stream = ctx->s_stream; SSL_SESSION * ssl_sess = NULL; if (events & BEV_EVENT_ERROR) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, ctx->mgr->logger); promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to original server failed."); } else if(events & BEV_EVENT_EOF) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); promise_failed(promise, FUTURE_ERROR_EXCEPTION, "original server closed."); } else if(events & BEV_EVENT_TIMEOUT) { promise_failed(promise, FUTURE_ERROR_TIMEOUT, NULL); } else if(events & BEV_EVENT_CONNECTED) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_NUM])); bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success //The reference count of the SSL_SESSION is not incremented, so no need to free. ssl_sess = SSL_get0_session(s_stream->ssl); up_session_set(s_stream->mgr->up_sess_cache, (struct sockaddr *)&(ctx->addr), ctx->addrlen, s_stream->client_hello->sni, ssl_sess); promise_success(promise, ctx); } ssl_connect_origin_ctx_free(ctx); return; } static void peek_chello_on_succ(future_result_t * result, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_get_ctx(p); struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. if(chello->sni==NULL) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_SNI])); } ctx->s_stream = ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL); ctx->bev = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_upstream, ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1); bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, p); bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only future_destroy(ctx->f_peek_chello); ctx->f_peek_chello = NULL; return; } static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO])); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream."); ssl_connect_origin_ctx_free(ctx); return; } extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, evutil_socket_t fd_downstream, struct event_base * evbase) { struct promise * p = future_to_promise(f); struct ssl_connect_origin_ctx * ctx = ALLOC(struct ssl_connect_origin_ctx, 1); int ret = 0; ctx->addrlen = sizeof(ctx->addr); ret = getpeername(fd_downstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen)); assert(ret == 0); ctx->fd_downstream = fd_downstream; ctx->fd_upstream = fd_upstream; ctx->evbase = evbase; ctx->mgr = mgr; promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free); ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p); ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); } /* * Called by OpenSSL when a new src SSL session is created. * OpenSSL increments the refcount before calling the callback and will * decrement it again if we return 0. Returning 1 will make OpenSSL skip * the refcount decrementing. In other words, return 0 if we did not * keep a pointer to the object (which we never do here). */ static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR); #ifdef HAVE_SSLV2 /* Session resumption seems to fail for SSLv2 with protocol * parsing errors, so we disable caching for SSLv2. */ if (SSL_version(ssl) == SSL2_VERSION) { return 0; } #endif /* HAVE_SSLV2 */ if (sess) { down_session_set(mgr->down_sess_cache, sess); } return 0; } /* * Called by OpenSSL when a src SSL session should be removed. * OpenSSL calls SSL_SESSION_free() after calling the callback; * we do not need to free the reference here. */ static void ossl_sessremove_cb(SSL_CTX * sslctx, SSL_SESSION * sess) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_CTX_get_ex_data(sslctx, SSL_EX_DATA_IDX_SSLMGR); assert(mgr != NULL); if (sess) { down_session_del(mgr->down_sess_cache, sess); } return; } /* * Called by OpenSSL when a src SSL session is requested by the client. OPENSSL_VERSION_NUMBER >= 0x10100000L required. */ static SSL_SESSION * ossl_sessget_cb(SSL * ssl, const unsigned char * id, int idlen, int * copy) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR); SSL_SESSION * sess; *copy = 0; /* SSL should not increment reference count of session */ sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache, id, idlen); return sess; } /* * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX. */ static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr) { SSL_CTX_set_options(sslctx, SSL_OP_ALL); #ifdef SSL_OP_TLS_ROLLBACK_BUG SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG); #endif /* SSL_OP_TLS_ROLLBACK_BUG */ #ifdef SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); #endif /* SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION */ #ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); #endif /* SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS */ #ifdef SSL_OP_NO_TICKET SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET); #endif /* SSL_OP_NO_TICKET */ #ifdef SSL_OP_NO_SSLv2 #ifdef HAVE_SSLV2 if (mgr->no_ssl2) { #endif /* HAVE_SSLV2 */ SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); #ifdef HAVE_SSLV2 } #endif /* HAVE_SSLV2 */ #endif /* !SSL_OP_NO_SSLv2 */ #ifdef HAVE_SSLV3 if (mgr->no_ssl3) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); } #endif /* HAVE_SSLV3 */ #ifdef HAVE_TLSV10 if (mgr->no_tls10) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1); } #endif /* HAVE_TLSV10 */ #ifdef HAVE_TLSV11 if (mgr->no_tls11) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1); } #endif /* HAVE_TLSV11 */ #ifdef HAVE_TLSV12 if (mgr->no_tls12) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2); } #endif /* HAVE_TLSV12 */ #ifdef SSL_OP_NO_COMPRESSION if (!mgr->sslcomp) { SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION); } #endif /* SSL_OP_NO_COMPRESSION */ } /* * Create and set up a new SSL_CTX instance for terminating SSL. * Set up all the necessary callbacks, the keyring, the keyring chain and key. */ static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt) { SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); if (!sslctx) return NULL; SSL * ssl = NULL; int ret = 0; sslctx_set_opts(sslctx, mgr); SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); //TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L if (mgr->ssl_min_version) { if (SSL_CTX_set_min_proto_version(sslctx, mgr->ssl_min_version) == 0 || SSL_CTX_set_max_proto_version(sslctx, mgr->ssl_max_version) == 0) { SSL_CTX_free(sslctx); return NULL; } } SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); SSL_CTX_set_session_id_context(sslctx, (const unsigned char *) mgr->ssl_session_context, sizeof(mgr->ssl_session_context)); if (mgr->dh) { SSL_CTX_set_tmp_dh(sslctx, mgr->dh); } else { SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); } if (mgr->ecdhcurve) { EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } else { EC_KEY * ecdh = ssl_ec_by_name(NULL); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } SSL_CTX_use_certificate(sslctx, crt->cert); SSL_CTX_use_PrivateKey(sslctx, crt->key); for (int i = 0; i < sk_X509_num(crt->chain); i++) { X509 * c = sk_X509_value(crt->chain, i); ssl_x509_refcount_inc(c); /* next call consumes a reference */ SSL_CTX_add_extra_chain_cert(sslctx, c); } ssl = SSL_new(sslctx); SSL_CTX_free(sslctx); // SSL_new() increments refcount sslctx = NULL; ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); assert(ret == 1); if (mgr->ssl_mode_release_buffers == 1) { /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); } return ssl; } void query_cert_ctx_free(struct ask_keyring_ctx * ctx) { X509_free(ctx->origin_crt); if (ctx->f_query_cert != NULL) { future_destroy(ctx->f_query_cert); ctx->f_query_cert = NULL; } //on success, bev_down and downstream has been transfered to caller by release** if (ctx->bev_down != NULL) { bufferevent_free(ctx->bev_down); } if (ctx->downstream != NULL) { ssl_stream_free(ctx->downstream); } return; } void query_cert_ctx_free_cb(void * p) { struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)p; query_cert_ctx_free(ctx); } struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result) { struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) result; struct ssl_stream * ret = ctx->downstream; ctx->downstream = NULL; return ret; } struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result) { struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) result; struct bufferevent * ret = ctx->bev_down; ctx->bev_down = NULL; return ret; } void ask_keyring_on_succ(void * result, void * user) { struct promise * p = (struct promise *) user; struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) promise_dettach_ctx(p); struct keyring * kyr = NULL; struct ssl_mgr * mgr = ctx->ssl_mgr; kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing. future_destroy(ctx->f_query_cert); ctx->f_query_cert = NULL; ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, kyr); ctx->bev_down = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl, BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); promise_success(p, ctx); query_cert_ctx_free(ctx); } void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user) { struct promise * p = (struct promise *) user; struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) promise_dettach_ctx(p); promise_failed(p, error, what); query_cert_ctx_free(ctx); return; } /* * Create a SSL stream for the incoming connection, based on the upstream. */ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base * evbase) { assert(upstream->dir == CONN_DIR_UPSTREAM); const char* sni=NULL; struct ask_keyring_ctx * ctx = ALLOC(struct ask_keyring_ctx, 1); ctx->keyring_id = keyring_id; ctx->ssl_mgr = mgr; ctx->fd_downstream = fd_downstream; ctx->evbase = evbase; if (upstream != NULL) { ctx->origin_ssl = upstream; ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl); sni=upstream->client_hello->sni; } struct promise * p = future_to_promise(f); promise_set_ctx(p, ctx, query_cert_ctx_free_cb); ctx->is_origin_crt_vaild = ssl_conn_verify_cert(mgr->trust_CA_store, upstream->ssl); if(!ctx->is_origin_crt_vaild) { ATOMIC_INC(&(mgr->stat_val[SSL_FAKE_CRT])); } ctx->f_query_cert = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p); key_keeper_async_ask(ctx->f_query_cert, mgr->keeper_of_keys, sni, keyring_id, ctx->origin_crt, ctx->is_origin_crt_vaild, evbase); return; } /* * Cleanly shut down an SSL socket. Libevent currently has no support for * cleanly shutting down an SSL socket so we work around that by using a * low-level event. This works for recent versions of OpenSSL. OpenSSL * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE * may or may not work. */ static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase) { struct ssl_shutdown_ctx * ctx = ALLOC(struct ssl_shutdown_ctx, 1); ctx->evbase = evbase; ctx->s_stream = s_stream; ctx->ev = NULL; ctx->retries = 0; return ctx; } static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx) { free(ctx); } /* * The shutdown socket event handler. This is either * scheduled as a timeout-only event, or as a fd read or * fd write event, depending on whether SSL_shutdown() * indicates it needs read or write on the socket. */ static void pxy_ssl_shutdown_cb(evutil_socket_t fd, short what, void * arg) { struct ssl_shutdown_ctx * ctx = (struct ssl_shutdown_ctx *) arg; struct timeval retry_delay = {0, 100}; void * logger = ctx->s_stream->mgr->logger; short want = 0; int rv = 0, sslerr = 0; if (ctx->ev) { event_free(ctx->ev); ctx->ev = NULL; } /* * Use the new (post-2008) semantics for SSL_shutdown() on a * non-blocking socket. SSL_shutdown() returns -1 and WANT_READ * if the other end's close notify was not received yet, and * WANT_WRITE it could not write our own close notify. * * This is a good collection of recent and relevant documents: * http://bugs.python.org/issue8108 */ rv = SSL_shutdown(ctx->s_stream->ssl); if (rv == 1) goto complete; if (rv != -1) { goto retry; } switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) { case SSL_ERROR_WANT_READ: want = EV_READ; goto retry; case SSL_ERROR_WANT_WRITE: want = EV_WRITE; goto retry; case SSL_ERROR_ZERO_RETURN: goto retry; case SSL_ERROR_SYSCALL: case SSL_ERROR_SSL: goto complete; default: TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() " "error %i. Closing fd.\n", sslerr); goto complete; } goto complete; retry: if (ctx->retries++ >= MAX_NET_RETRIES) { TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " "Max retries reached. Closing fd.\n"); goto complete; } ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); if (ctx->ev) { event_add(ctx->ev, &retry_delay); } else { TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " "Cannot create event. Closing fd.\n"); } return; complete: if(ctx->s_stream->dir==CONN_DIR_DOWNSTREAM) { ATOMIC_INC(&(ctx->s_stream->mgr->stat_val[SSL_DOWN_CLOSED])); } else { ATOMIC_INC(&(ctx->s_stream->mgr->stat_val[SSL_UP_CLOSED])); } ssl_stream_free(ctx->s_stream); evutil_closesocket(fd); ssl_shutdown_ctx_free(ctx); } /* * Cleanly shutdown an SSL session on file descriptor fd using low-level * file descriptor readiness events on event base evbase. * Guarantees that SSL and the corresponding SSL_CTX are freed and the * socket is closed, eventually, or in the case of fatal errors, immediately. */ void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, evutil_socket_t fd) { struct ssl_shutdown_ctx * sslshutctx = NULL; assert(fd==s_stream->_do_not_use.fd); sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); pxy_ssl_shutdown_cb(fd, 0, sslshutctx); }