/* * SSL stream implementation. * Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise. * Author: zhengchao@iie.ac.cn * Create: 2018-8-17 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include int SSL_EX_DATA_IDX_SSLMGR; int SSL_EX_DATA_IDX_VERIFIED; int SSL_PEER_CERT_VERIFY_PASSED=1; int SSL_PEER_CERT_VERIFY_FAILED=0; #define MAX_NET_RETRIES 50 /* * Default cipher suite spec. * Use 'openssl ciphers -v spec' to see what ciphers are effectively enabled * by a cipher suite spec with a given version of OpenSSL. */ #define DFLT_CIPHERS "ALL:-aNULL" /* * Default elliptic curve for EC cipher suites. */ #define DFLT_CURVE "prime256v1" enum ssl_stream_stat { SSL_UP_NEW, SSL_UP_ERR, SSL_UP_CLOSING, SSL_UP_CLOSED, SSL_UP_DIRTY_CLOSED, SSL_UP_CACHE_SZ, SSL_UP_CACHE_QUERY, SSL_UP_CACHE_HIT, SSL_DOWN_NEW, SSL_DOWN_ERR, SSL_DOWN_CLOSING, SSL_DOWN_CLOSED, SSL_DOWN_DIRTY_CLOSED, SSL_DOWN_CACHE_SZ, SSL_DOWN_CACHE_QUERY, SSL_DOWN_CACHE_HIT, SSL_DOWN_TICKET_NEW, SSL_DOWN_TICKET_REUSE, SSL_DOWN_TICKET_NOTFOUND, SSL_DOWN_TIKCET_QUERY, SSL_NO_CHELLO, SSL_NO_SNI, SSL_FAKE_CRT, SSL_STAT_MAX }; struct session_ticket_key { size_t size; unsigned char name[16]; unsigned char hmac_key[32]; unsigned char aes_key[32]; } ; struct ssl_mgr { unsigned int sslcomp; unsigned int no_ssl2; unsigned int no_ssl3; unsigned int no_tls10; unsigned int no_tls11; unsigned int no_tls12; unsigned int no_sessticket; CONST_SSL_METHOD * (* sslmethod)(void); //Parameter of SSL_CTX_new int ssl_min_version, ssl_max_version; char ssl_session_context[8]; unsigned int cache_slots; unsigned int sess_expire_seconds; struct sess_cache * down_sess_cache; struct sess_cache * up_sess_cache; struct session_ticket_key ticket_key; char default_ciphers[TFE_SYMBOL_MAX]; DH * dh; char * ecdhcurve; char * crl_url; uint8_t ssl_mode_release_buffers; char trust_CA_file[TFE_PATH_MAX]; char trust_CA_dir[TFE_PATH_MAX]; X509_STORE * trust_CA_store; struct key_keeper * key_keeper; struct event_base * ev_base_gc; struct event * gcev; unsigned int log_master_key; char master_key_file[TFE_PATH_MAX]; FILE* fp_master_key; void * logger; screen_stat_handle_t fs_handle; long long stat_val[SSL_STAT_MAX]; int fs_id[SSL_STAT_MAX]; }; struct __ssl_stream_debug { evutil_socket_t fd; }; struct ssl_stream { enum tfe_conn_dir dir; SSL * ssl; struct ssl_mgr * mgr; union { struct ssl_chello * client_hello; //dir=upstream, a little weird, which send by downstream client. struct keyring * keyring; //dir=downstream. }; struct __ssl_stream_debug _do_not_use; int is_peer_cert_verify_passed; }; struct peek_client_hello_ctx { struct ssl_chello* chello; unsigned char sni_peek_retries; /* max 64 SNI parse retries */ struct event * ev; struct event_base * evbase; void * logger; }; struct ssl_connect_server_ctx { struct bufferevent * bev; struct ssl_stream * s_stream; struct ssl_mgr * mgr; struct sockaddr_storage addr; socklen_t addrlen; void * logger; evutil_socket_t fd_upstream; evutil_socket_t fd_downstream; struct event_base * evbase; struct future * f_peek_chello; }; struct ssl_connect_client_ctx { int keyring_id; struct ssl_stream * origin_ssl; X509 * origin_crt; int is_origin_crt_verify_passed; struct ssl_mgr * ssl_mgr; evutil_socket_t fd_downstream; struct event_base * evbase; struct future * f_ask_keyring; struct bufferevent * bev_down; struct ssl_stream * downstream; }; /* * SSL shutdown context. */ struct ssl_shutdown_ctx { struct ssl_stream * s_stream; struct event_base * evbase; struct event * ev; struct ssl_mgr* mgr; enum tfe_conn_dir dir; unsigned int retries; }; struct fs_spec { enum ssl_stream_stat id; const char* name; }; /* * Garbage collection handler. */ static void ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) { struct ssl_mgr *mgr=(struct ssl_mgr *)arg; int i=0; ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT])); ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT])); for(i=0;ifs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i]))); } if(mgr->log_master_key && mgr->fp_master_key) { fflush(mgr->fp_master_key); } return; } void ssl_stat_init(struct ssl_mgr * mgr) { int i=0; const char* spec[SSL_STAT_MAX]={0}; spec[SSL_UP_NEW]="ussl_new"; spec[SSL_UP_ERR]="ussl_err"; spec[SSL_UP_CLOSING]="ussl_clsing"; spec[SSL_UP_CLOSED]="ussl_clsed"; spec[SSL_UP_DIRTY_CLOSED]="ussl_dirty_cls"; spec[SSL_UP_CACHE_SZ]="usess_cache"; spec[SSL_UP_CACHE_QUERY]="usess_query"; spec[SSL_UP_CACHE_HIT]="usess_hitcnt"; spec[SSL_DOWN_NEW]="dssl_new"; spec[SSL_DOWN_ERR]="dssl_err"; spec[SSL_DOWN_CLOSING]="dssl_clsing"; spec[SSL_DOWN_CLOSED]="dssl_clsed"; spec[SSL_DOWN_DIRTY_CLOSED]="dssl_dirty_cls"; spec[SSL_DOWN_CACHE_SZ]="dsess_cache"; spec[SSL_DOWN_CACHE_QUERY]="dcache_query"; spec[SSL_DOWN_CACHE_HIT]="dsess_hitcnt"; if(!mgr->no_sessticket) { spec[SSL_DOWN_TICKET_NEW]="dtkt_new"; spec[SSL_DOWN_TICKET_REUSE]="dtkt_reuse"; spec[SSL_DOWN_TICKET_NOTFOUND]="dtkt_notfnd"; spec[SSL_DOWN_TIKCET_QUERY]="dtkt_query"; } spec[SSL_NO_CHELLO]="ssl_no_chlo"; spec[SSL_NO_SNI]="ssl_no_sni"; spec[SSL_FAKE_CRT]="ssl_fk_crt"; for(i=0;ifs_id[i]=FS_register(mgr->fs_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,spec[i]); } } int value=mgr->fs_id[SSL_UP_CACHE_HIT]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); value=mgr->fs_id[SSL_UP_CACHE_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_register_ratio(mgr->fs_handle, mgr->fs_id[SSL_UP_CACHE_HIT], mgr->fs_id[SSL_UP_CACHE_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, "usess_hit"); value=mgr->fs_id[SSL_DOWN_CACHE_HIT]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); value=mgr->fs_id[SSL_DOWN_CACHE_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_register_ratio(mgr->fs_handle, mgr->fs_id[SSL_DOWN_CACHE_HIT], mgr->fs_id[SSL_DOWN_CACHE_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, "dsess_hit"); if(!mgr->no_sessticket) { value=mgr->fs_id[SSL_DOWN_TIKCET_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); value=mgr->fs_id[SSL_DOWN_TICKET_REUSE]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_register_ratio(mgr->fs_handle, mgr->fs_id[SSL_DOWN_TICKET_REUSE], mgr->fs_id[SSL_DOWN_TIKCET_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, "dtkt_hit"); } struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here. mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr); evtimer_add(mgr->gcev, &gc_delay); return; } static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt); static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, evutil_socket_t fd); static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr); struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) { struct peek_client_hello_ctx* ctx= (struct peek_client_hello_ctx*) result; struct ssl_chello * p = ctx->chello; ctx->chello=NULL; return p; } struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir, struct ssl_chello * client_hello, struct keyring * kyr) { int ret = 0; struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1); s_stream->dir = dir; s_stream->mgr = mgr; s_stream->_do_not_use.fd = fd; assert(ret == 0); switch (dir) { case CONN_DIR_DOWNSTREAM: ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_NEW])); s_stream->ssl = downstream_ssl_create(mgr, kyr); s_stream->keyring = kyr; break; case CONN_DIR_UPSTREAM: ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_NEW])); s_stream->ssl = upstream_ssl_create(mgr, client_hello, fd); s_stream->client_hello = client_hello; break; default: assert(0); } return s_stream; } static void ssl_stream_free(struct ssl_stream * s_stream) { SSL_free(s_stream->ssl); s_stream->ssl = NULL; switch (s_stream->dir) { case CONN_DIR_DOWNSTREAM: if (s_stream->keyring != NULL) { key_keeper_free_keyring(s_stream->keyring); s_stream->keyring = NULL; } ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED])); break; case CONN_DIR_UPSTREAM: if (s_stream->client_hello != NULL) { ssl_chello_free(s_stream->client_hello); s_stream->client_hello = NULL; } ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_CLOSED])); break; default: assert(0); } s_stream->mgr = NULL; free(s_stream); return; } static int sslver_str2num(const char * version_str) { int sslversion = -1; assert(OPENSSL_VERSION_NUMBER >= 0x10100000L); /* * Support for SSLv2 and the corresponding SSLv2_method(), * SSLv2_server_method() and SSLv2_client_method() functions were * removed in OpenSSL 1.1.0. */ if (!strcmp(version_str, "ssl3")) { sslversion = SSL3_VERSION; } else if (!strcmp(version_str, "tls10") || !strcmp(version_str, "tls1")) { sslversion = TLS1_VERSION; } else if (!strcmp(version_str, "tls11")) { sslversion = TLS1_1_VERSION; } else if (!strcmp(version_str, "tls12")) { sslversion = TLS1_2_VERSION; } else if (!strcmp(version_str, "tls13")) { sslversion = TLS1_3_VERSION; } else { sslversion = -1; } return sslversion; } static void log_ssl_master_key(SSL* ssl, int fd, tfe_conn_dir dir, FILE* fp) { char* key_str=NULL; key_str=ssl_ssl_masterkey_to_str(ssl); char time_str[TFE_SYMBOL_MAX]; time_t now=time(NULL); tfe_thread_safe_ctime(&now, time_str, sizeof(time_str)); struct tfe_stream_addr* addr=tfe_stream_addr_create_by_fd(fd, dir); char* addr_string=tfe_stream_addr_to_str(addr); fprintf(fp, "#%s %s %s\n%s\r", time_str, tfe_stream_conn_dir_to_str(dir), addr_string, key_str); free(key_str); tfe_stream_addr_free(addr); free(addr_string); return; } void ssl_manager_destroy(struct ssl_mgr * mgr) { if (mgr->key_keeper) { key_keeper_destroy(mgr->key_keeper); } if (mgr->trust_CA_store) { X509_STORE_free(mgr->trust_CA_store); mgr->trust_CA_store = NULL; } if(mgr->down_sess_cache) { ssl_sess_cache_destroy(mgr->down_sess_cache); } if(mgr->up_sess_cache) { ssl_sess_cache_destroy(mgr->up_sess_cache); } if(mgr->gcev) { event_free(mgr->gcev); } if(mgr->fp_master_key) { fclose(mgr->fp_master_key); } free(mgr); } struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, struct event_base * ev_base_gc, void * logger) { unsigned char key_name[]="!mesalab-tfe3a~&"; unsigned char aes_key_def[]={0xC5,0xAC,0xC1,0xA6,0xB2,0xBB,0xCA,0xC7,0xE3,0xBE,0xE3,0xB2,0xC6,0xA3,0xB1,0xB9 ,0xA3,0xAC,0xB6,0xF8,0xCA,0xC7,0xD1,0xDB,0xBE,0xA6,0xC0,0xEF,0xD3,0xD0,0xB9,0x84}; unsigned char hmac_key_def[]={0xD6,0xC8,0xD0,0xF2,0xC8,0xCB,0xBF,0xDA,0xBA,0xCD,0xBF,0xC6,0xBC,0xBC,0xA3,0xAC ,0xC8,0xCB,0xC0,0xE0,0xC9,0xE7,0xBB,0xE1,0xB5,0xC4,0xBB,0xF9,0xCA,0xAF,0x19,0x84}; struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1); int ret = 0; char version_str[TFE_SYMBOL_MAX]; mgr->logger = logger; mgr->ev_base_gc=ev_base_gc; MESA_load_profile_string_def(ini_profile, section, "ssl_min_version", version_str, sizeof(version_str), "ssl3"); mgr->ssl_min_version = sslver_str2num(version_str); MESA_load_profile_string_def(ini_profile, section, "ssl_max_version", version_str, sizeof(version_str), "tls12"); mgr->ssl_max_version = sslver_str2num(version_str); if (mgr->ssl_min_version < 0) { TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str); goto error_out; } ret=ssl_init(); if(ret<0) { TFE_LOG_ERROR(logger, "OpenSSL global init failed."); goto error_out; } //tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0. mgr->sslmethod = TLS_method; SSL_EX_DATA_IDX_SSLMGR = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL); SSL_EX_DATA_IDX_VERIFIED = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL); MESA_load_profile_uint_def(ini_profile, section, "ssl_compression", &(mgr->sslcomp), 1); MESA_load_profile_uint_def(ini_profile, section, "no_ssl2", &(mgr->no_ssl2), 1); MESA_load_profile_uint_def(ini_profile, section, "no_ssl3", &(mgr->no_ssl3), 1); MESA_load_profile_uint_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0); MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers, sizeof(mgr->default_ciphers), DFLT_CIPHERS); MESA_load_profile_uint_def(ini_profile, section, "no_session_ticket", &(mgr->no_sessticket), 0); MESA_load_profile_uint_def(ini_profile, section, "session_cache_slots", &(mgr->cache_slots), 4 * 1024 * 1024); MESA_load_profile_uint_def(ini_profile, section, "session_cache_expire_seconds", &(mgr->sess_expire_seconds), 30 * 60); mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); //Reference to NGINX: http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_ticket_key //Support key rotation in futher. memcpy(mgr->ticket_key.name,key_name, sizeof(mgr->ticket_key.name)); memcpy(mgr->ticket_key.aes_key, aes_key_def, sizeof(mgr->ticket_key.aes_key)); memcpy(mgr->ticket_key.hmac_key, hmac_key_def, sizeof(mgr->ticket_key.hmac_key)); mgr->key_keeper = key_keeper_init(ini_profile, "key_keeper", logger); if (mgr->key_keeper == NULL) { TFE_LOG_ERROR(logger, "Certificate Manager initiate failed."); goto error_out; } mgr->trust_CA_store = X509_STORE_new(); if (mgr->trust_CA_store == NULL) { TFE_LOG_ERROR(logger, "Failed at creating X509_STORE"); goto error_out; } ret = X509_STORE_set_default_paths(mgr->trust_CA_store); if (ret == 0) { TFE_LOG_ERROR(logger, "Failed at setting default paths for X509_STORE."); goto error_out; } MESA_load_profile_string_def(ini_profile, section, "trust_CA_file", mgr->trust_CA_file, sizeof(mgr->trust_CA_file), "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"); MESA_load_profile_string_def(ini_profile, section, "trust_CA_dir", mgr->trust_CA_dir, sizeof(mgr->trust_CA_dir), ""); ret = X509_STORE_load_locations(mgr->trust_CA_store, strlen(mgr->trust_CA_file) > 0 ? mgr->trust_CA_file : NULL, strlen(mgr->trust_CA_dir) > 0 ? mgr->trust_CA_dir : NULL); if (ret == 0) { TFE_LOG_ERROR(logger, "Failed at setting load locations for X509_STORE"); goto error_out; } memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context)); MESA_load_profile_uint_def(ini_profile, section, "log_master_key", &(mgr->log_master_key), 0); MESA_load_profile_string_def(ini_profile, section, "key_log_file", mgr->master_key_file, sizeof(mgr->master_key_file), "./sslkeylog.log"); if(mgr->log_master_key) { mgr->fp_master_key=fopen(mgr->master_key_file, "a"); if(mgr->fp_master_key==NULL) { TFE_LOG_ERROR(logger, "Failed at open master key log file %s", mgr->master_key_file); mgr->log_master_key=0; } } mgr->fs_handle=tfe_proxy_get_fs_handle(); ssl_stat_init(mgr); return mgr; error_out: ssl_manager_destroy(mgr); return NULL; } int ssl_conn_verify_cert(X509_STORE * store, const SSL * ssl, char** error_string) { int ret = 0, err_code=0; char *subj=NULL, *issuer=NULL; STACK_OF(X509) * cert_chain = SSL_get_peer_cert_chain(ssl); if (cert_chain == NULL) { // The peer certificate chain is not necessarily available after reusing a session, in which case a NULL pointer is returned. return 1; } X509_STORE_CTX * ctx = X509_STORE_CTX_new(); X509 * cert = sk_X509_value(cert_chain, 0); ret = X509_STORE_CTX_init(ctx, store, cert, cert_chain); assert(ret == 1); //If a complete chain can be built and validated this function returns 1, otherwise it return zero or negtive code. ret = X509_verify_cert(ctx); if(ret!=1) { err_code=X509_STORE_CTX_get_error(ctx); subj=ssl_x509_subject(cert); issuer=ssl_x509_issuer(cert); asprintf(error_string, "%s : subject - %s issuer - %s" , X509_verify_cert_error_string(err_code) , subj , issuer); free(subj); free(issuer); } X509_STORE_CTX_free(ctx); return (ret == 1); } void peek_client_hello_ctx_free(struct peek_client_hello_ctx * _ctx) { event_free(_ctx->ev); _ctx->ev = NULL; if(_ctx->chello!=NULL) { ssl_chello_free(_ctx->chello); _ctx->chello=NULL; } free(_ctx); } void peek_client_hello_ctx_free_cb(void * p) { struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)p; return peek_client_hello_ctx_free(_ctx); } static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) { struct promise * promise = (struct promise *) arg; //use promise_get_ctx instead of promise_dettach_ctx for try more times. struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *) promise_get_ctx(promise); enum chello_parse_result chello_status=CHELLO_PARSE_INVALID_FORMAT; struct ssl_chello* chello=NULL; const char * reason = NULL; unsigned char buf[2048]; ssize_t n = 0; n = recv(fd, buf, sizeof(buf), MSG_PEEK); if (n == -1) { TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection\n"); goto failed; } if (n == 0) { goto failed; } chello=ssl_chello_parse(buf,n, &chello_status); switch(chello_status) { case CHELLO_PARSE_SUCCESS: { promise_dettach_ctx(promise); ctx->chello=chello; promise_success(promise, ctx); peek_client_hello_ctx_free(ctx); break; } case CHELLO_PARSE_NOT_ENOUGH_BUFF: { ssl_chello_free(chello); chello=NULL; if (ctx->sni_peek_retries++ > MAX_NET_RETRIES) { TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n"); reason = "too many peek retries"; goto failed; } /* ssl_tls_clienthello_parse indicates that we * should retry later when we have more data, and we * haven't reached the maximum retry count yet. * Reschedule this event as timeout-only event in * order to prevent busy looping over the read event. * Because we only peeked at the pending bytes and * never actually read them, fd is still ready for * reading now. We use 25 * 0.2 s = 5 s timeout. */ struct timeval retry_delay = {0, 100}; event_free(ctx->ev); ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); assert(ctx->ev != NULL); event_add(ctx->ev, &retry_delay); break; } case CHELLO_PARSE_INVALID_FORMAT: { ssl_chello_free(chello); chello=NULL; TFE_LOG_ERROR(ctx->logger, "Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); reason = "see no client hello"; goto failed; break; } default: assert(0); } return; failed: promise_dettach_ctx(promise); promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason); peek_client_hello_ctx_free(ctx); return; } static void ssl_async_peek_client_hello(struct future * future, evutil_socket_t fd, struct event_base * evbase, void * logger) { struct promise * p = future_to_promise(future); struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1); ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); ctx->logger = logger; event_add(ctx->ev, NULL); promise_set_ctx(p, (void *) ctx, peek_client_hello_ctx_free_cb); return; } /* * Create new SSL context for outgoing connections to the original destination. * If hostname sni is provided, use it for Server Name Indication. */ static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, evutil_socket_t fd) { SSL_CTX * sslctx = NULL; SSL * ssl = NULL; SSL_SESSION * sess = NULL; sslctx = SSL_CTX_new(mgr->sslmethod()); sslctx_set_opts(sslctx, mgr); int ret=0; if(chello->cipher_suites!=NULL) { //SSL_CTX_set_cipher_list() and SSL_set_cipher_list() return 1 if any cipher could be selected and 0 on complete failure. ret=SSL_CTX_set_cipher_list(sslctx, chello->cipher_suites); if(ret==0) { TFE_LOG_ERROR(mgr->logger, "SSL_CTX_set_cipher_list %s failed.", chello->cipher_suites); SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); } } else { ret=SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); } if (mgr->ssl_min_version) { if (SSL_CTX_set_min_proto_version(sslctx, MAX(chello->min_version.ossl_format, mgr->ssl_min_version)) == 0 || SSL_CTX_set_max_proto_version(sslctx, MIN(chello->max_version.ossl_format, mgr->ssl_max_version)) == 0) { SSL_CTX_free(sslctx); return NULL; } } SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); ssl = SSL_new(sslctx); SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ if (!ssl) { return NULL; } if (chello->sni) { SSL_set_tlsext_host_name(ssl, chello->sni); } if (chello->alpn) { ret=SSL_set_alpn_protos(ssl, (unsigned char*)chello->alpn, strlen(chello->alpn)); assert(0); } /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); struct sockaddr_storage addr; socklen_t addrlen = sizeof(struct sockaddr_storage); ret = getpeername(fd, (struct sockaddr *) (&addr), &addrlen); if(ret == 0) { /* session resuming based on remote endpoint address and port */ sess = up_session_get(mgr->up_sess_cache, (struct sockaddr *) &addr, addrlen, chello->sni); if (sess) { SSL_set_session(ssl, sess); /* increments sess refcount */ SSL_SESSION_free(sess); } } return ssl; } void ssl_connect_server_ctx_free(struct ssl_connect_server_ctx * ctx) { if (ctx->s_stream != NULL) { ssl_stream_free(ctx->s_stream); } if (ctx->bev != NULL) { bufferevent_free(ctx->bev); ctx->bev = NULL; } if (ctx->f_peek_chello != NULL) { future_destroy(ctx->f_peek_chello); ctx->f_peek_chello = NULL; } free(ctx); return; } void wrap_ssl_connect_server_ctx_free(void *p) { struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *)p; ssl_connect_server_ctx_free(ctx); } struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result) { struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result; struct ssl_stream * ret = ctx->s_stream; ctx->s_stream = NULL; //giveup ownership return ret; } struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result) { struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result; struct bufferevent * ret = ctx->bev; ctx->bev = NULL; //giveup ownership return ret; } void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, void* logger) { unsigned long sslerr=0; int fd=bufferevent_getfd(bev); char* addr_string=tfe_string_addr_create_by_fd(fd, dir); /* Can happen for socket errs, ssl errs; * may happen for unclean ssl socket shutdowns. */ sslerr = bufferevent_get_openssl_error(bev); if (!errno && !sslerr) { /* We have disabled notification for unclean shutdowns * so this should not happen; log a warning. */ TFE_LOG_ERROR(logger,"Warning: Spurious error from " "bufferevent (errno=0,sslerr=0)\n"); } else if (ERR_GET_REASON(sslerr) == SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE) { /* these can happen due to client cert auth, * only log error if debugging is activated */ TFE_LOG_ERROR(logger,"Handshake Error from bufferevent of ssl %s %s: " "%i:%s %lu:%i:%s:%i:%s:%i:%s", tfe_stream_conn_dir_to_str(dir), addr_string, errno, errno ? strerror(errno) : "-", sslerr, ERR_GET_REASON(sslerr), sslerr ? ERR_reason_error_string(sslerr) : "-", ERR_GET_LIB(sslerr), sslerr ? ERR_lib_error_string(sslerr) : "-", ERR_GET_FUNC(sslerr), sslerr ? ERR_func_error_string(sslerr) : "-"); while ((sslerr = bufferevent_get_openssl_error(bev))) { TFE_LOG_ERROR(logger,"Additional SSL error: " "%lu:%i:%s:%i:%s:%i:%s", sslerr, ERR_GET_REASON(sslerr), ERR_reason_error_string(sslerr), ERR_GET_LIB(sslerr), ERR_lib_error_string(sslerr), ERR_GET_FUNC(sslerr), ERR_func_error_string(sslerr)); } } else { /* real errors */ TFE_LOG_ERROR(logger,"Error from bufferevent of ssl %s %s: " "%i:%s %lu:%i:%s:%i:%s:%i:%s", tfe_stream_conn_dir_to_str(dir), addr_string, errno, errno ? strerror(errno) : "-", sslerr, ERR_GET_REASON(sslerr), sslerr ? ERR_reason_error_string(sslerr) : "-", ERR_GET_LIB(sslerr), sslerr ? ERR_lib_error_string(sslerr) : "-", ERR_GET_FUNC(sslerr), sslerr ? ERR_func_error_string(sslerr) : "-"); while ((sslerr = bufferevent_get_openssl_error(bev))) { TFE_LOG_ERROR(logger,"Additional SSL error: " "%lu:%i:%s:%i:%s:%i:%s\n", sslerr, ERR_GET_REASON(sslerr), ERR_reason_error_string(sslerr), ERR_GET_LIB(sslerr), ERR_lib_error_string(sslerr), ERR_GET_FUNC(sslerr), ERR_func_error_string(sslerr)); } } free(addr_string); } /* * Callback for meta events on the up- and downstream connection bufferevents. * Called when EOF has been reached, a connection has been made, and on errors. */ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, void * arg) { struct promise * p = (struct promise *) arg; struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p); struct ssl_stream * s_stream = ctx->s_stream; struct ssl_mgr* mgr=s_stream->mgr; SSL_SESSION * ssl_sess = NULL; char* error_string=NULL; if (events & BEV_EVENT_ERROR) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, ctx->mgr->logger); promise_failed(p, FUTURE_ERROR_EXCEPTION, "connect to original server failed."); } else if(events & BEV_EVENT_EOF) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); promise_failed(p, FUTURE_ERROR_EXCEPTION, "original server closed."); } else if(events & BEV_EVENT_TIMEOUT) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); promise_failed(p, FUTURE_ERROR_TIMEOUT, NULL); } else if(events & BEV_EVENT_CONNECTED) { bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success if(!SSL_session_reused(s_stream->ssl)) { s_stream->is_peer_cert_verify_passed = ssl_conn_verify_cert(s_stream->mgr->trust_CA_store, s_stream->ssl, &error_string); if(s_stream->is_peer_cert_verify_passed) { //ONLY verified session is cacheable. //The reference count of the SSL_SESSION is not incremented, so no need to free. ssl_sess = SSL_get0_session(s_stream->ssl); up_session_set(mgr->up_sess_cache, (struct sockaddr *)&(ctx->addr), ctx->addrlen, s_stream->client_hello->sni, ssl_sess); } else { ATOMIC_INC(&(mgr->stat_val[SSL_FAKE_CRT])); char* addr_str=tfe_string_addr_create_by_fd(ctx->fd_upstream, CONN_DIR_UPSTREAM); TFE_LOG_INFO(mgr->logger, "Fake Cert %s %s : %s", addr_str, ctx->s_stream->client_hello->sni, error_string); free(addr_str); free(error_string); } } else { //Do not perform cert check on reused session. s_stream->is_peer_cert_verify_passed=1; } if(mgr->log_master_key) { log_ssl_master_key(s_stream->ssl, ctx->fd_upstream, CONN_DIR_UPSTREAM, mgr->fp_master_key); } promise_success(p, ctx); } wrap_ssl_connect_server_ctx_free(ctx); return; } static void peek_chello_on_succ(future_result_t * result, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_get_ctx(p); struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. if(chello->sni==NULL) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_SNI])); } ctx->s_stream = ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL); ctx->bev = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_upstream, ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1); bufferevent_setcb(ctx->bev, NULL, NULL, ssl_server_connected_eventcb, p); bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only future_destroy(ctx->f_peek_chello); ctx->f_peek_chello = NULL; return; } static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p); ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO])); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream."); wrap_ssl_connect_server_ctx_free(ctx); return; } extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, evutil_socket_t fd_downstream, struct event_base * evbase) { struct promise * p = future_to_promise(f); struct ssl_connect_server_ctx * ctx = ALLOC(struct ssl_connect_server_ctx, 1); int ret = 0; ctx->addrlen = sizeof(ctx->addr); ret = getpeername(fd_upstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen)); if(ret!=0) { ssl_connect_server_ctx_free(ctx); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream fd closed"); return; } ctx->fd_downstream = fd_downstream; ctx->fd_upstream = fd_upstream; ctx->evbase = evbase; ctx->mgr = mgr; promise_set_ctx(p, ctx, wrap_ssl_connect_server_ctx_free); ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p); ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); } static int ossl_session_ticket_key_callback(SSL *ssl_conn, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc) { const EVP_MD *digest=EVP_sha256(); const EVP_CIPHER *cipher=EVP_aes_256_cbc(); size_t size=32; struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl_conn, SSL_EX_DATA_IDX_SSLMGR); assert(mgr!=NULL); struct session_ticket_key* key=&(mgr->ticket_key); unsigned char buf[33]={0}; ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY])); if (enc == 1) { /* encrypt session ticket */ if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1) { TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed"); return -1; } if (EVP_EncryptInit_ex(ectx, cipher, NULL, key[0].aes_key, iv) != 1) { TFE_LOG_ERROR(mgr->logger, "EVP_EncryptInit_ex() failed"); return -1; } if (HMAC_Init_ex(hctx, key[0].hmac_key, size, digest, NULL) != 1) { TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed"); return -1; } memcpy(name, key[0].name, 16); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW])); return 1; } else { /* decrypt session ticket */ if(0!=memcmp(name, key[0].name, 16)) { TFE_LOG_INFO(mgr->logger, "ssl session ticket decrypt, key: \"%*s\" not found" ,tfe_hexdump(buf, name ,16)-buf, buf); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND])); return 0; } if (HMAC_Init_ex(hctx, key[0].hmac_key, size, digest, NULL) != 1) { TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed"); return -1; } if (EVP_DecryptInit_ex(ectx, cipher, NULL, key[0].aes_key, iv) != 1) { TFE_LOG_ERROR(mgr->logger, "EVP_DecryptInit_ex() failed"); return -1; } ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_REUSE])); //Todo: Implement key expire. return 1; //Return 1 indicates that the ctx and hctx have been set and the session can continue on those parameters. } } /* * Called by OpenSSL when a new src SSL session is created. * OpenSSL increments the refcount before calling the callback and will * decrement it again if we return 0. Returning 1 will make OpenSSL skip * the refcount decrementing. In other words, return 0 if we did not * keep a pointer to the object (which we never do here). */ static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR); #ifdef HAVE_SSLV2 /* Session resumption seems to fail for SSLv2 with protocol * parsing errors, so we disable caching for SSLv2. */ if (SSL_version(ssl) == SSL2_VERSION) { return 0; } #endif /* HAVE_SSLV2 */ if (sess) { down_session_set(mgr->down_sess_cache, sess); } return 0; } /* * Called by OpenSSL when a src SSL session should be removed. * OpenSSL calls SSL_SESSION_free() after calling the callback; * we do not need to free the reference here. */ static void ossl_sessremove_cb(SSL_CTX * sslctx, SSL_SESSION * sess) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_CTX_get_ex_data(sslctx, SSL_EX_DATA_IDX_SSLMGR); assert(mgr != NULL); if (sess) { down_session_del(mgr->down_sess_cache, sess); } return; } /* * Called by OpenSSL when a src SSL session is requested by the client. OPENSSL_VERSION_NUMBER >= 0x10100000L required. */ static SSL_SESSION * ossl_sessget_cb(SSL * ssl, const unsigned char * id, int idlen, int * copy) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR); SSL_SESSION * sess; *copy = 0; /* SSL should not increment reference count of session */ sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache, id, idlen); return sess; } /* * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX. */ static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr) { SSL_CTX_set_options(sslctx, SSL_OP_ALL); #ifdef SSL_OP_TLS_ROLLBACK_BUG SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG); #endif /* SSL_OP_TLS_ROLLBACK_BUG */ #ifdef SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); #endif /* SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION */ #ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); #endif /* SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS */ if (mgr->no_ssl2) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); } if (mgr->no_ssl3) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); } if (mgr->no_tls10) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1); } if (mgr->no_tls11) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1); } if (mgr->no_tls12) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2); } if(mgr->no_sessticket) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET); } if (!mgr->sslcomp) { SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION); } } /* * Create and set up a new SSL_CTX instance for terminating SSL. * Set up all the necessary callbacks, the keyring, the keyring chain and key. */ static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt) { SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); if (!sslctx) return NULL; SSL * ssl = NULL; int ret = 0; sslctx_set_opts(sslctx, mgr); SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); //TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L if (mgr->ssl_min_version) { if (SSL_CTX_set_min_proto_version(sslctx, mgr->ssl_min_version) == 0 || SSL_CTX_set_max_proto_version(sslctx, mgr->ssl_max_version) == 0) { SSL_CTX_free(sslctx); return NULL; } } SSL_CTX_sess_set_new_cb(sslctx, ossl_sessnew_cb); SSL_CTX_sess_set_remove_cb(sslctx, ossl_sessremove_cb); SSL_CTX_sess_set_get_cb(sslctx, ossl_sessget_cb); if(!mgr->no_sessticket) { SSL_CTX_set_tlsext_ticket_key_cb(sslctx, ossl_session_ticket_key_callback); } SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); SSL_CTX_set_session_id_context(sslctx, (const unsigned char *) mgr->ssl_session_context, sizeof(mgr->ssl_session_context)); ret = SSL_CTX_set_ex_data(sslctx, SSL_EX_DATA_IDX_SSLMGR, mgr); assert(ret == 1); if (mgr->dh) { SSL_CTX_set_tmp_dh(sslctx, mgr->dh); } else { SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); } if (mgr->ecdhcurve) { EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } else { EC_KEY * ecdh = ssl_ec_by_name(NULL); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } SSL_CTX_use_certificate(sslctx, crt->cert); SSL_CTX_use_PrivateKey(sslctx, crt->key); for (int i = 0; i < sk_X509_num(crt->chain); i++) { X509 * c = sk_X509_value(crt->chain, i); ssl_x509_refcount_inc(c); /* next call consumes a reference */ SSL_CTX_add_extra_chain_cert(sslctx, c); } ssl = SSL_new(sslctx); SSL_CTX_free(sslctx); // SSL_new() increments refcount sslctx = NULL; ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr); assert(ret == 1); if (mgr->ssl_mode_release_buffers == 1) { /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); } return ssl; } void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx) { X509_free(ctx->origin_crt); ctx->origin_crt=NULL; if (ctx->f_ask_keyring != NULL) { future_destroy(ctx->f_ask_keyring); ctx->f_ask_keyring = NULL; } //on success, bev_down and downstream has been transfered to caller by release** if (ctx->bev_down != NULL) { bufferevent_free(ctx->bev_down); } if (ctx->downstream != NULL) { ssl_stream_free(ctx->downstream); } return; } void wrap_ssl_connect_client_ctx_free(void * p) { struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *)p; ssl_connect_client_ctx_free(ctx); } struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result) { struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result; struct ssl_stream * ret = ctx->downstream; ctx->downstream = NULL; return ret; } struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result) { struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result; struct bufferevent * ret = ctx->bev_down; ctx->bev_down = NULL; return ret; } static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, void * arg) { struct promise * p = (struct promise *) arg; struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p); struct ssl_stream * s_stream = ctx->downstream; struct ssl_mgr* mgr=s_stream->mgr; SSL_SESSION * ssl_sess = NULL; if (events & BEV_EVENT_ERROR) { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); ssl_stream_log_error(bev, CONN_DIR_DOWNSTREAM, mgr->logger); promise_failed(p, FUTURE_ERROR_EXCEPTION, "connect to client failed."); } else if(events & BEV_EVENT_EOF) { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); promise_failed(p, FUTURE_ERROR_EXCEPTION, "client side closed."); } else if(events & BEV_EVENT_TIMEOUT) { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); promise_failed(p, FUTURE_ERROR_TIMEOUT, NULL); } else if(events & BEV_EVENT_CONNECTED) { bufferevent_disable(ctx->bev_down, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev_down, NULL, NULL, NULL, NULL); //leave a clean bev for on_success if(mgr->log_master_key) { log_ssl_master_key(ctx->downstream->ssl, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key); } promise_success(p, ctx); } ssl_connect_client_ctx_free(ctx); return; } void ask_keyring_on_succ(void * result, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_get_ctx(p); struct keyring * kyr = NULL; struct ssl_mgr * mgr = ctx->ssl_mgr; kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing. ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, kyr); ctx->bev_down = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl, BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p); bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only future_destroy(ctx->f_ask_keyring); ctx->f_ask_keyring = NULL; } void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p); promise_failed(p, error, what); ssl_connect_client_ctx_free(ctx); return; } /* * Create a SSL stream for the incoming connection, based on the upstream. */ extern void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, evutil_socket_t fd_downstream, int keyring_id, struct event_base * evbase) { assert(upstream->dir == CONN_DIR_UPSTREAM); int * previous_verfiy_result=NULL; const char* sni=NULL; struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1); ctx->keyring_id = keyring_id; ctx->ssl_mgr = mgr; ctx->fd_downstream = fd_downstream; ctx->evbase = evbase; if (upstream != NULL) { ctx->origin_ssl = upstream; ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl); sni=upstream->client_hello->sni; } struct promise * p = future_to_promise(f); promise_set_ctx(p, ctx, wrap_ssl_connect_client_ctx_free); ctx->f_ask_keyring = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p); ctx->is_origin_crt_verify_passed = upstream->is_peer_cert_verify_passed; key_keeper_async_ask(ctx->f_ask_keyring, mgr->key_keeper, sni, keyring_id, ctx->origin_crt, ctx->is_origin_crt_verify_passed, evbase); return; } /* * Cleanly shut down an SSL socket. Libevent currently has no support for * cleanly shutting down an SSL socket so we work around that by using a * low-level event. This works for recent versions of OpenSSL. OpenSSL * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE * may or may not work. */ static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase) { struct ssl_shutdown_ctx * ctx = ALLOC(struct ssl_shutdown_ctx, 1); ctx->evbase = evbase; ctx->s_stream = s_stream; ctx->ev = NULL; ctx->mgr = s_stream->mgr; ctx->dir = s_stream->dir; ctx->retries = 0; ctx->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_INC(&(ctx->mgr->stat_val[SSL_DOWN_CLOSING])) : ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_CLOSING])); return ctx; } static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx) { ctx->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_DEC(&(ctx->mgr->stat_val[SSL_DOWN_CLOSING])) : ATOMIC_DEC(&(ctx->mgr->stat_val[SSL_UP_CLOSING])); memset(ctx, 0, sizeof(struct ssl_shutdown_ctx)); free(ctx); } /* * The shutdown socket event handler. This is either * scheduled as a timeout-only event, or as a fd read or * fd write event, depending on whether SSL_shutdown() * indicates it needs read or write on the socket. */ static void pxy_ssl_shutdown_cb(evutil_socket_t fd, short what, void * arg) { struct ssl_shutdown_ctx * ctx = (struct ssl_shutdown_ctx *) arg; struct timeval retry_delay = {0, 100}; void * logger = ctx->s_stream->mgr->logger; struct ssl_mgr* mgr=ctx->s_stream->mgr; short want = 0; int rv = 0, sslerr = 0; char tmp[1024]; if (ctx->ev) { event_free(ctx->ev); ctx->ev = NULL; } /* * Use the new (post-2008) semantics for SSL_shutdown() on a * non-blocking socket. SSL_shutdown() returns -1 and WANT_READ * if the other end's close notify was not received yet, and * WANT_WRITE it could not write our own close notify. * * This is a good collection of recent and relevant documents: * http://bugs.python.org/issue8108 */ rv = SSL_shutdown(ctx->s_stream->ssl); if (rv == 1) goto complete; if (rv != -1) { goto retry; } switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv))) { case SSL_ERROR_WANT_READ: want = EV_READ; goto retry; case SSL_ERROR_WANT_WRITE: want = EV_WRITE; goto retry; case SSL_ERROR_ZERO_RETURN: goto retry; case SSL_ERROR_SYSCALL: case SSL_ERROR_SSL: goto complete; default: TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() " "error %i. Closing fd.\n", sslerr); goto complete; } goto complete; retry: if (ctx->retries++ >= MAX_NET_RETRIES) { /* struct tfe_stream_addr* addr=tfe_stream_addr_create_by_fd(fd, ctx->s_stream->dir); char* addr_string=tfe_stream_addr_to_str(addr); TFE_LOG_ERROR(logger, "Failed to shutdown %s SSL connection cleanly: %s " "Max retries reached. Closing fd %d.", tfe_stream_conn_dir_to_str(ctx->s_stream->dir), addr_string, fd); tfe_stream_addr_free(addr); free(addr_string); */ if(ctx->s_stream->dir==CONN_DIR_DOWNSTREAM) { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_DIRTY_CLOSED])); } else { ATOMIC_INC(&(mgr->stat_val[SSL_UP_DIRTY_CLOSED])); } goto complete; } ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx); if (ctx->ev) { event_add(ctx->ev, &retry_delay); } else { TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: " "Cannot create event. Closing fd %d.", fd); } return; complete: ssl_stream_free(ctx->s_stream); evutil_closesocket(fd); ssl_shutdown_ctx_free(ctx); } /* * Cleanly shutdown an SSL session on file descriptor fd using low-level * file descriptor readiness events on event base evbase. * Guarantees that SSL and the corresponding SSL_CTX are freed and the * socket is closed, eventually, or in the case of fatal errors, immediately. */ void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, evutil_socket_t fd) { struct ssl_shutdown_ctx * sslshutctx = NULL; assert(fd==s_stream->_do_not_use.fd); sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); pxy_ssl_shutdown_cb(fd, 0, sslshutctx); }