/* * SSL stream implementation. * Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise. * Author: zhengchao@iie.ac.cn * Create: 2018-8-17 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int SSL_CTX_EX_DATA_IDX_SSLMGR; static int SSL_EX_DATA_IDX_SSLSTREAM; static unsigned int ssl_debug; #define MAX_NET_RETRIES 50 #define LATENCY_WARNING_THRESHOLD_MS 1000 /* * Default cipher suite spec. * Use 'openssl ciphers -v spec' to see what ciphers are effectively enabled * by a cipher suite spec with a given version of OpenSSL. */ #define DFLT_CIPHERS "ALL:-aNULL" static unsigned char SSL_ALPN_HTTP_1_1[]={8, 'h', 't', 't', 'p', '/', '1', '.', '1',0}; static unsigned char SSL_ALPN_HTTP_2[]={2, 'h','2',0}; /* * Default elliptic curve for EC cipher suites. */ #define DFLT_CURVE "prime256v1" extern long long certstore_is_unavailable; enum ssl_stream_stat { SSL_UP_NEW, SSL_UP_ERR, SSL_UP_ERR_NO_CIPHER, SSL_UP_ERR_UNSUPPORT_PROTO, SSL_UP_CLOSING, SSL_UP_CLOSED, SSL_UP_DIRTY_CLOSED, SSL_UP_CACHE_SZ, SSL_UP_CACHE_QUERY, SSL_UP_CACHE_HIT, SSL_DOWN_NEW, SSL_DOWN_ERR, SSL_DOWN_ERR_NO_CERT, SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK, SSL_DOWN_CLOSING, SSL_DOWN_CLOSED, SSL_DOWN_DIRTY_CLOSED, SSL_DOWN_CACHE_SZ, SSL_DOWN_CACHE_QUERY, SSL_DOWN_CACHE_HIT, SSL_DOWN_TICKET_NEW, SSL_DOWN_TICKET_REUSE, SSL_DOWN_TICKET_NOTFOUND, SSL_DOWN_TIKCET_QUERY, SSL_NO_CHELLO, SSL_NO_SNI, SSL_FAKE_CRT, KEY_KEEPER_CACHE_SIZE, KEY_KEEPER_ASK, KEY_KEEPER_ISSUE, SSL_SVC_PINNING, SSL_SVC_MAUTH, SSL_SVC_CT_CERT, SSL_SVC_EV_CERT, SSL_SVC_APP_NOT_PINNING, SSL_SVC_TRUSTED_CERT, SSL_STAT_MAX }; struct ssl_mgr { unsigned int sslcomp; unsigned int no_ssl2; unsigned int no_ssl3; unsigned int no_tls10; unsigned int no_tls11; unsigned int no_tls12; unsigned int no_sesscache; unsigned int no_sessticket; unsigned int no_alpn; unsigned int no_cert_verify; unsigned int no_mirror_client_cipher_suite; CONST_SSL_METHOD * (* sslmethod)(void); //Parameter of SSL_CTX_new int ssl_min_version, ssl_max_version; char ssl_session_context[8]; unsigned int sess_cache_slots; unsigned int sess_expire_seconds; unsigned int svc_cache_slots; unsigned int svc_expire_seconds; unsigned int svc_fail_as_pinning_cnt; unsigned int svc_fail_as_proto_err_cnt; unsigned int svc_succ_as_app_not_pinning_cnt; unsigned int svc_cnt_time_window; struct sess_cache * down_sess_cache; struct sess_cache * up_sess_cache; struct sess_ticket_box * down_stek_box; struct ssl_service_cache* svc_cache; ssl_stream_new_hook* on_new_upstream_cb; void* upstream_cb_param; struct sess_ticket_key ticket_key; char default_ciphers[TFE_SYMBOL_MAX]; DH * dh; char * ecdhcurve; char * crl_url; unsigned int trusted_cert_load_local; struct cert_store_param cert_verify_param; uint8_t ssl_mode_release_buffers; char trusted_cert_file[TFE_PATH_MAX]; char trusted_cert_dir[TFE_PATH_MAX]; struct ssl_trusted_cert_storage * trust_CA_store; struct key_keeper * key_keeper; struct event_base * ev_base_gc; struct event * gcev; unsigned int log_master_key; char master_key_file[TFE_PATH_MAX]; FILE* fp_master_key; void * logger; screen_stat_handle_t fs_handle; long long stat_val[SSL_STAT_MAX]; int fs_id[SSL_STAT_MAX]; }; struct __ssl_stream_debug { evutil_socket_t fd; }; struct ssl_bypass_condition { char bypass_ev_cert; char bypass_ct_cert; char bypass_mutual_auth; char bypass_cert_pinning; }; struct ssl_upstream_parts { struct cert_verify_param verify_param; struct cert_verify_result verify_result; char block_fake_cert; struct ssl_service_status svc_status; enum ssl_stream_action action; int apln_enabled; int keyring_for_trusted; int keyring_for_untrusted; struct ssl_chello * client_hello; uint8_t is_server_cert_verify_passed; }; struct ssl_downstream_parts { struct keyring * keyring; }; struct ssl_stream { enum tfe_conn_dir dir; SSL * ssl; struct ssl_mgr * mgr; union { struct ssl_upstream_parts up_parts; struct ssl_downstream_parts down_parts; }; int ssl_min_version, ssl_max_version, negotiated_version; const unsigned char* alpn_selected; //reference to SSL_ALPN_HTTP_2/SSL_ALPN_HTTP_1_1 struct tfe_stream* tcp_stream; uint64_t connect_latency_ms; struct ssl_stream* peer; socklen_t peer_addrlen; struct sockaddr_storage peer_addr; struct __ssl_stream_debug _do_not_use; enum ssl_stream_error error; }; struct peek_client_hello_ctx { struct ssl_chello* chello; unsigned char sni_peek_retries; /* max 64 SNI parse retries */ struct event * ev; struct event_base * evbase; void * logger; }; struct ssl_connect_server_ctx { struct bufferevent * bev; struct ssl_stream * s_stream; struct ssl_mgr * mgr; struct sockaddr_storage addr; socklen_t addrlen; void * logger; evutil_socket_t fd_upstream; evutil_socket_t fd_downstream; struct tfe_stream* tcp_stream; struct future * f_peek_chello; struct timespec start,end; }; struct ssl_connect_client_ctx { struct tfe_stream* tcp_stream; struct ssl_stream * peer; X509 * origin_crt; int is_origin_crt_verify_passed; struct ssl_mgr * ssl_mgr; evutil_socket_t fd_downstream; struct future * f_ask_keyring; struct bufferevent * bev_down; struct ssl_stream * downstream; struct timespec start,end; }; /* * SSL shutdown context. */ struct ssl_shutdown_ctx { struct ssl_stream * s_stream; struct event_base * evbase; struct event * ev; struct ssl_mgr* mgr; enum tfe_conn_dir dir; unsigned int retries; }; struct fs_spec { enum ssl_stream_stat id; const char* name; }; unsigned int is_ssl_debug() { return ssl_debug; } int sslver_str2num(const char * version_str) { int sslversion = -1; assert(OPENSSL_VERSION_NUMBER >= 0x10100000L); /* * Support for SSLv2 and the corresponding SSLv2_method(), * SSLv2_server_method() and SSLv2_client_method() functions were * removed in OpenSSL 1.1.0. */ if (!strcasecmp(version_str, "ssl3")||!strcasecmp(version_str, "SSLv3")||!strcasecmp(version_str, "sslv3.0")) { sslversion = SSL3_VERSION; } else if (!strcasecmp(version_str, "tls10") || !strcasecmp(version_str, "tls1") || !strcasecmp(version_str, "TLSv1.0") || !strcasecmp(version_str, "TLSv1")) { sslversion = TLS1_VERSION; } else if (!strcasecmp(version_str, "tls11")||!strcasecmp(version_str, "TLSv1.1")) { sslversion = TLS1_1_VERSION; } else if (!strcasecmp(version_str, "tls12")||!strcasecmp(version_str, "TLSv1.2")) { sslversion = TLS1_2_VERSION; } else if (!strcasecmp(version_str, "tls13")||!strcasecmp(version_str, "TLSv1.3")) { sslversion = TLS1_3_VERSION; } else { sslversion = -1; } return sslversion; } const char* ssl_stream_get_error_string(enum ssl_stream_error error) { const char* ssl_err_str[__SSL_STREAM_R_MAX]; ssl_err_str[SSL_STREAM_R_NO_ERROR]="OK"; ssl_err_str[SSL_STREAM_R_SERVER_CLOSED]="server-side closed"; ssl_err_str[SSL_STREAM_R_CLIENT_CLOSED]="client-side closed"; ssl_err_str[SSL_STREAM_R_CONNECT_SERVER_TIMEOUT]="server-side timeout"; ssl_err_str[SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT]="client-side timeout"; ssl_err_str[SSL_STREAM_R_SERVER_PROTOCOL_ERROR]="server-side protocol errors"; ssl_err_str[SSL_STREAM_R_CLIENT_PROTOCOL_ERROR]="client-side protocol errors"; return ssl_err_str[error]; } /* * Garbage collection handler. */ static void ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) { struct ssl_mgr *mgr=(struct ssl_mgr *)arg; int i=0; if(!mgr->no_sesscache) { ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT])); ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT])); } struct key_keeper_stat keeper_stat = { 0 }; key_keeper_statistic(mgr->key_keeper, &keeper_stat); mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times; mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue; mgr->stat_val[KEY_KEEPER_CACHE_SIZE]=keeper_stat.cached_num; struct ssl_service_cache_statistics svc_stat; memset(&svc_stat, 0, sizeof(svc_stat)); ssl_service_cache_stat(mgr->svc_cache, &svc_stat); mgr->stat_val[SSL_SVC_PINNING]=svc_stat.pinning_cli_cnt; mgr->stat_val[SSL_SVC_MAUTH]=svc_stat.mutual_auth_cli_cnt; mgr->stat_val[SSL_SVC_CT_CERT]=svc_stat.ct_srv_cnt; mgr->stat_val[SSL_SVC_EV_CERT]=svc_stat.ev_srv_cnt; mgr->stat_val[SSL_SVC_APP_NOT_PINNING]=svc_stat.app_not_pinning_cnt; mgr->stat_val[SSL_SVC_TRUSTED_CERT]=svc_stat.trusted_cert_cnt; for(i=0;ifs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i]))); } if(mgr->log_master_key && mgr->fp_master_key) { fflush(mgr->fp_master_key); } return; } void ssl_stat_init(struct ssl_mgr * mgr) { int i=0; const char* spec[SSL_STAT_MAX]={0}; spec[SSL_UP_NEW]="ussl_new"; spec[SSL_UP_ERR]="ussl_err"; spec[SSL_UP_ERR_NO_CIPHER]="ussl_e_ciph"; spec[SSL_UP_ERR_UNSUPPORT_PROTO]="ussl_e_prt"; spec[SSL_UP_CLOSING]="ussl_clsing"; spec[SSL_UP_CLOSED]="ussl_clsd"; spec[SSL_UP_DIRTY_CLOSED]="ussl_dt_cls"; spec[SSL_UP_CACHE_SZ]="usess_cache"; spec[SSL_UP_CACHE_QUERY]="usess_query"; spec[SSL_UP_CACHE_HIT]="usess_hitn"; spec[SSL_DOWN_NEW]="dssl_new"; spec[SSL_DOWN_ERR]="dssl_err"; spec[SSL_DOWN_ERR_NO_CERT]="dssl_e_cert"; spec[SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK]="dssl_e_fb"; spec[SSL_DOWN_CLOSING]="dssl_clsing"; spec[SSL_DOWN_CLOSED]="dssl_clsd"; spec[SSL_DOWN_DIRTY_CLOSED]="dssl_dt_cls"; spec[SSL_DOWN_CACHE_SZ]="dsess_cache"; spec[SSL_DOWN_CACHE_QUERY]="dcache_query"; spec[SSL_DOWN_CACHE_HIT]="dsess_hitn"; if(!mgr->no_sessticket) { spec[SSL_DOWN_TICKET_NEW]="dtkt_new"; spec[SSL_DOWN_TICKET_REUSE]="dtkt_reuse"; spec[SSL_DOWN_TICKET_NOTFOUND]="dtkt_notfnd"; spec[SSL_DOWN_TIKCET_QUERY]="dtkt_query"; } spec[SSL_NO_CHELLO]="ssl_no_chlo"; spec[SSL_NO_SNI]="ssl_no_sni"; spec[SSL_FAKE_CRT]="ssl_fk_crt"; spec[KEY_KEEPER_ASK]="kyr_ask"; spec[KEY_KEEPER_ISSUE]="kyr_new"; spec[KEY_KEEPER_CACHE_SIZE]="kyr_cache"; spec[SSL_SVC_PINNING]="ssl_pinning"; spec[SSL_SVC_MAUTH]="ssl_mauth"; spec[SSL_SVC_CT_CERT]="ssl_ct_crt"; spec[SSL_SVC_EV_CERT]="ssl_ev_crt"; spec[SSL_SVC_APP_NOT_PINNING]="app_no_pinning"; spec[SSL_SVC_TRUSTED_CERT]="trusted_cert_nums"; for(i=0;ifs_id[i]=FS_register(mgr->fs_handle, FS_STYLE_FIELD, FS_CALC_CURRENT,spec[i]); } } int value=mgr->fs_id[SSL_UP_CACHE_HIT]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); value=mgr->fs_id[SSL_UP_CACHE_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_register_ratio(mgr->fs_handle, mgr->fs_id[SSL_UP_CACHE_HIT], mgr->fs_id[SSL_UP_CACHE_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, "usess_hit"); value=mgr->fs_id[SSL_DOWN_CACHE_HIT]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); value=mgr->fs_id[SSL_DOWN_CACHE_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_register_ratio(mgr->fs_handle, mgr->fs_id[SSL_DOWN_CACHE_HIT], mgr->fs_id[SSL_DOWN_CACHE_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, "dsess_hit"); if(!mgr->no_sessticket) { value=mgr->fs_id[SSL_DOWN_TIKCET_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); value=mgr->fs_id[SSL_DOWN_TICKET_REUSE]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_register_ratio(mgr->fs_handle, mgr->fs_id[SSL_DOWN_TICKET_REUSE], mgr->fs_id[SSL_DOWN_TIKCET_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, "dtkt_hit"); } struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here. mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr); evtimer_add(mgr->gcev, &gc_delay); return; } static void downstream_ossl_init(struct ssl_stream* s_stream); static int upstream_ossl_init(struct ssl_stream* s_stream); static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr); struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) { struct peek_client_hello_ctx* ctx= (struct peek_client_hello_ctx*) result; struct ssl_chello * p = ctx->chello; ctx->chello=NULL; return p; } struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir, struct ssl_chello * client_hello, struct keyring * kyr, struct ssl_stream * peer, struct tfe_stream* tcp_stream) { UNUSED int ret = 0; struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1); s_stream->dir = dir; s_stream->mgr = mgr; s_stream->_do_not_use.fd = fd; s_stream->ssl_max_version=mgr->ssl_max_version; s_stream->ssl_min_version=mgr->ssl_min_version; s_stream->peer=peer; s_stream->tcp_stream=tcp_stream; ret = getpeername(fd, (struct sockaddr *) (&s_stream->peer_addr), &(s_stream->peer_addrlen)); switch (dir) { case CONN_DIR_DOWNSTREAM: assert(peer!=NULL); ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_NEW])); s_stream->down_parts.keyring = kyr; s_stream->peer=peer; break; case CONN_DIR_UPSTREAM: ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_NEW])); s_stream->up_parts.verify_param.no_verify_cn=1; s_stream->up_parts.client_hello = client_hello; s_stream->ssl_min_version=client_hello->min_version.ossl_format; s_stream->ssl_max_version=client_hello->max_version.ossl_format; break; default: assert(0); } return s_stream; } static void ssl_stream_free(struct ssl_stream * s_stream) { if(s_stream->ssl) { SSL_free(s_stream->ssl); s_stream->ssl = NULL; } switch (s_stream->dir) { case CONN_DIR_DOWNSTREAM: if (s_stream->down_parts.keyring != NULL) { key_keeper_free_keyring(s_stream->down_parts.keyring); s_stream->down_parts.keyring = NULL; } ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED])); break; case CONN_DIR_UPSTREAM: if (s_stream->up_parts.client_hello != NULL) { ssl_chello_free(s_stream->up_parts.client_hello); s_stream->up_parts.client_hello = NULL; } ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_CLOSED])); break; default: assert(0); } if(s_stream->alpn_selected) { s_stream->alpn_selected=NULL; } s_stream->mgr = NULL; free(s_stream); return; } static void log_ssl_master_key(struct ssl_stream *s_stream, tfe_conn_dir dir, FILE* fp) { char* key_str=NULL; key_str=ssl_ssl_masterkey_to_str(s_stream->ssl); char time_str[TFE_SYMBOL_MAX]; time_t now=time(NULL); tfe_thread_safe_ctime(&now, time_str, sizeof(time_str)); fprintf(fp, "#%s %s %s\n%s\n", time_str, tfe_stream_conn_dir_to_str(dir), s_stream->tcp_stream->str_stream_info ? s_stream->tcp_stream->str_stream_info : NULL, key_str); free(key_str); return; } void ssl_manager_destroy(struct ssl_mgr * mgr) { if (mgr->key_keeper) { key_keeper_destroy(mgr->key_keeper); } if (mgr->trust_CA_store) { ssl_trusted_cert_storage_destroy(mgr->trust_CA_store); mgr->trust_CA_store = NULL; } if(mgr->down_sess_cache) { ssl_sess_cache_destroy(mgr->down_sess_cache); } if(mgr->up_sess_cache) { ssl_sess_cache_destroy(mgr->up_sess_cache); } if(mgr->gcev) { event_free(mgr->gcev); } if(mgr->fp_master_key) { fclose(mgr->fp_master_key); } free(mgr); } struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, struct event_base * ev_base_gc, struct key_keeper * key_keeper, void * logger) { unsigned int stek_group_num = 0; unsigned int stek_rotation_time = 0; struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1); int ret = 0; char ja3_table_name[TFE_STRING_MAX] = {0}; char version_str[TFE_SYMBOL_MAX] = {}; mgr->logger = logger; mgr->ev_base_gc=ev_base_gc; if (ssl_mid_cert_kafka_logger_create(ini_profile, section)) { goto error_out; } MESA_load_profile_uint_def(ini_profile, section, "ssl_debug", &(ssl_debug), 0); MESA_load_profile_string_def(ini_profile, section, "ssl_min_version", version_str, sizeof(version_str), "ssl3"); mgr->ssl_min_version = sslver_str2num(version_str); if (mgr->ssl_min_version < 0) { TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str); goto error_out; } MESA_load_profile_string_def(ini_profile, section, "ssl_max_version", version_str, sizeof(version_str), "tls12"); mgr->ssl_max_version = sslver_str2num(version_str); if (mgr->ssl_max_version < 0) { TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str); goto error_out; } ret=ssl_init(); if(ret<0) { TFE_LOG_ERROR(logger, "OpenSSL global init failed."); goto error_out; } //tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0. mgr->sslmethod = TLS_method; SSL_CTX_EX_DATA_IDX_SSLMGR = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL); SSL_EX_DATA_IDX_SSLSTREAM = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL); MESA_load_profile_uint_def(ini_profile, section, "ssl_compression", &(mgr->sslcomp), 1); MESA_load_profile_uint_def(ini_profile, section, "no_ssl2", &(mgr->no_ssl2), 1); MESA_load_profile_uint_def(ini_profile, section, "no_ssl3", &(mgr->no_ssl3), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0); MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers, sizeof(mgr->default_ciphers), DFLT_CIPHERS); MESA_load_profile_uint_def(ini_profile, section, "no_session_cache", &(mgr->no_sesscache), 0); MESA_load_profile_uint_def(ini_profile, section, "no_session_ticket", &(mgr->no_sessticket), 0); MESA_load_profile_uint_def(ini_profile, section, "no_alpn", &(mgr->no_alpn), 0); MESA_load_profile_uint_def(ini_profile, section, "no_cert_verify", &(mgr->no_cert_verify), 0); MESA_load_profile_uint_def(ini_profile, section, "no_mirror_client_cipher_suite", &(mgr->no_mirror_client_cipher_suite), 0); MESA_load_profile_uint_def(ini_profile, section, "session_cache_slots", &(mgr->sess_cache_slots), 4 * 1024 * 1024); MESA_load_profile_uint_def(ini_profile, section, "session_cache_expire_seconds", &(mgr->sess_expire_seconds), 30 * 60); if(!mgr->no_sesscache) { mgr->up_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); mgr->down_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); } MESA_load_profile_uint_def(ini_profile, section, "stek_group_num", &stek_group_num, 1); MESA_load_profile_uint_def(ini_profile, section, "stek_rotation_time", &stek_rotation_time, 3600); if(!mgr->no_sessticket) { mgr->down_stek_box = sess_ticket_box_create(ev_base_gc, stek_group_num, stek_rotation_time, logger); } MESA_load_profile_uint_def(ini_profile, section, "service_cache_slots", &(mgr->svc_cache_slots), 4 * 1024 * 1024); MESA_load_profile_uint_def(ini_profile, section, "service_cache_expire_seconds", &(mgr->svc_expire_seconds), 5 * 60); MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_as_pinning_cnt", &(mgr->svc_fail_as_pinning_cnt), 4); MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_as_proto_err_cnt", &(mgr->svc_fail_as_proto_err_cnt), 5); MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_time_window", &(mgr->svc_cnt_time_window), 30); MESA_load_profile_string_def(ini_profile, section, "ssl_ja3_table", ja3_table_name, sizeof(ja3_table_name), "PXY_SSL_FINGERPRINT"); mgr->svc_cache=ssl_service_cache_create(mgr->svc_cache_slots, mgr->svc_expire_seconds, mgr->svc_fail_as_pinning_cnt, mgr->svc_fail_as_proto_err_cnt, mgr->svc_cnt_time_window, ja3_table_name); if (mgr->svc_cache == NULL) { TFE_LOG_ERROR(logger, "Failed to create service cache"); goto error_out; } mgr->key_keeper = key_keeper; MESA_load_profile_uint_def(ini_profile, section, "trusted_cert_load_local", &(mgr->trusted_cert_load_local), 1); MESA_load_profile_uint_def(ini_profile, section, "check_cert_crl", &(mgr->cert_verify_param.check_crl), 0); if(mgr->trusted_cert_load_local)//Other wise, use policy defined trusted CA file. { MESA_load_profile_string_def(ini_profile, section, "trusted_cert_file", mgr->trusted_cert_file, sizeof(mgr->trusted_cert_file), "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"); MESA_load_profile_string_def(ini_profile, section, "trusted_cert_dir", mgr->trusted_cert_dir, sizeof(mgr->trusted_cert_dir), "./resource/tfe/trusted_storage"); } mgr->trust_CA_store = ssl_trusted_cert_storage_create(mgr->trusted_cert_file, mgr->trusted_cert_dir, &(mgr->cert_verify_param)); if (mgr->trust_CA_store == NULL) { TFE_LOG_ERROR(logger, "Failed at creating X509_STORE"); goto error_out; } memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context)); MESA_load_profile_uint_def(ini_profile, section, "log_master_key", &(mgr->log_master_key), 0); MESA_load_profile_string_def(ini_profile, section, "key_log_file", mgr->master_key_file, sizeof(mgr->master_key_file), "./sslkeylog.log"); if(mgr->log_master_key) { mgr->fp_master_key=fopen(mgr->master_key_file, "a"); if(mgr->fp_master_key==NULL) { TFE_LOG_ERROR(logger, "Failed at open master key log file %s", mgr->master_key_file); mgr->log_master_key=0; } } mgr->fs_handle=tfe_proxy_get_fs_handle(); ssl_stat_init(mgr); return mgr; error_out: ssl_manager_destroy(mgr); return NULL; } void ssl_manager_set_new_upstream_cb(struct ssl_mgr * mgr, ssl_stream_new_hook* new_upstream_cb, void* u_para) { mgr->on_new_upstream_cb=new_upstream_cb; mgr->upstream_cb_param=u_para; return; } void peek_client_hello_ctx_free(struct peek_client_hello_ctx * _ctx) { event_free(_ctx->ev); _ctx->ev = NULL; if(_ctx->chello!=NULL) { ssl_chello_free(_ctx->chello); _ctx->chello=NULL; } free(_ctx); } void peek_client_hello_ctx_free_cb(void * p) { struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)p; return peek_client_hello_ctx_free(_ctx); } static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) { struct promise * promise = (struct promise *) arg; //use promise_get_ctx instead of promise_dettach_ctx for try more times. struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *) promise_get_ctx(promise); enum chello_parse_result chello_status=CHELLO_PARSE_INVALID_FORMAT; struct ssl_chello* chello=NULL; const char * reason = NULL; unsigned char buf[2048]; ssize_t n = 0; n = recv(fd, buf, sizeof(buf), MSG_PEEK); if (n == -1) { char* addr_string=tfe_string_addr_create_by_fd(fd, CONN_DIR_DOWNSTREAM); TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection, addr_string is %s, errno is %d, errmsg is %s\n", addr_string, errno, strerror(errno)); /* after log, reset errno */ errno = 0; free(addr_string); goto failed; } if (n == 0) { goto failed; } chello=ssl_chello_parse(buf, n, &chello_status); switch(chello_status) { case CHELLO_PARSE_SUCCESS: { if (is_ssl_debug()) { char *addr = tfe_string_addr_create_by_fd(fd, CONN_DIR_DOWNSTREAM); struct ssl_ja3 *fingerprint = ssl_ja3_generate_fingerprint(buf, n); TFE_LOG_INFO(ctx->logger, "ja3:%s, hash:%s addr:%s\n", fingerprint ? fingerprint->ja3 : "null", fingerprint ? fingerprint->ja3_hash : "null", addr); ssl_ja3_free(fingerprint); free(addr); } promise_dettach_ctx(promise); ctx->chello=chello; promise_success(promise, ctx); peek_client_hello_ctx_free(ctx); break; } case CHELLO_PARSE_NOT_ENOUGH_BUFF: { ssl_chello_free(chello); chello=NULL; if (ctx->sni_peek_retries++ > MAX_NET_RETRIES) { TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n"); reason = "too many peek retries"; goto failed; } /* ssl_tls_clienthello_parse indicates that we * should retry later when we have more data, and we * haven't reached the maximum retry count yet. * Reschedule this event as timeout-only event in * order to prevent busy looping over the read event. * Because we only peeked at the pending bytes and * never actually read them, fd is still ready for * reading now. We use 25 * 0.2 s = 5 s timeout. */ struct timeval retry_delay = {0, 100}; event_del(ctx->ev); event_free(ctx->ev); ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); assert(ctx->ev != NULL); event_add(ctx->ev, &retry_delay); break; } case CHELLO_PARSE_INVALID_FORMAT: { ssl_chello_free(chello); chello=NULL; TFE_LOG_ERROR(ctx->logger, "Peeking did not yield a (truncated) ClientHello message, aborting connection\n"); reason = "see no client hello"; goto failed; break; } default: assert(0); } return; failed: promise_dettach_ctx(promise); promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason); peek_client_hello_ctx_free(ctx); return; } static void ssl_async_peek_client_hello(struct future * f, evutil_socket_t fd, struct event_base * evbase, void * logger) { struct promise * p = future_to_promise(f); struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1); ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p); ctx->evbase = evbase; ctx->logger = logger; promise_set_ctx(p, (void *) ctx, peek_client_hello_ctx_free_cb); event_add(ctx->ev, NULL); return; } int ossl_client_cert_cb(SSL *ssl, X509 **x509, EVP_PKEY **pkey) { struct ssl_stream* s_upstream=NULL; s_upstream=(struct ssl_stream*)SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM); s_upstream->up_parts.svc_status.is_mutual_auth=1; ssl_service_cache_write(s_upstream->mgr->svc_cache, s_upstream->up_parts.client_hello, s_upstream->tcp_stream, &s_upstream->up_parts.svc_status); return 0; } /* * Create new SSL context for outgoing connections to the original destination. * If hostname sni is provided, use it for Server Name Indication. */ static int upstream_ossl_init(struct ssl_stream* s_stream) { SSL_CTX * sslctx = NULL; SSL * ssl = NULL; SSL_SESSION * sess = NULL; struct ssl_mgr * mgr = s_stream->mgr; struct ssl_chello * chello=s_stream->up_parts.client_hello; sslctx = SSL_CTX_new(mgr->sslmethod()); if (sslctx == NULL) { TFE_LOG_ERROR(mgr->logger, "ssl ctx new failed."); return -1; } sslctx_set_opts(sslctx, mgr); int ret=0; char common_cipher[TFE_STRING_MAX]={0}, tls13_cipher[TFE_STRING_MAX]={0}; if(chello->cipher_suites) { ssl_cipher_suites_to_name(chello->cipher_suites, chello->cipher_suites_len, common_cipher, sizeof(common_cipher), tls13_cipher, sizeof(tls13_cipher)); } if(strlen(common_cipher)>0) { //SSL_CTX_set_cipher_list() and SSL_set_cipher_list() return 1 if any cipher could be selected and 0 on complete failure. ret=SSL_CTX_set_cipher_list(sslctx, common_cipher); if(ret==0) { TFE_LOG_ERROR(mgr->logger, "ssl ctx set cipher list %s failed.", common_cipher); SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); } } else { ret=SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); } if(strlen(tls13_cipher)>0 && s_stream->ssl_max_version==TLS1_3_VERSION) { //SSL_CTX_set_ciphersuites(sslctx, tls13_cipher); } if (SSL_CTX_set_min_proto_version(sslctx, s_stream->ssl_min_version) == 0) { SSL_CTX_free(sslctx); TFE_LOG_ERROR(mgr->logger, "ssl set min proto version failed %d.", s_stream->ssl_min_version); return -1; } if (SSL_CTX_set_max_proto_version(sslctx, s_stream->ssl_max_version) == 0) { SSL_CTX_free(sslctx); TFE_LOG_ERROR(mgr->logger, "ssl set max proto version failed %d.", s_stream->ssl_max_version); return -1; } SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); SSL_CTX_set_client_cert_cb(sslctx, ossl_client_cert_cb); if(mgr->no_sesscache) { SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_OFF); } else { SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_NO_INTERNAL); /* session resuming based on remote endpoint address and port */ sess = up_session_get(mgr->up_sess_cache, (struct sockaddr *) &(s_stream->peer_addr), s_stream->peer_addrlen, chello->sni, s_stream->ssl_min_version, s_stream->ssl_max_version); if (sess) { ret=SSL_CTX_add_session(sslctx, sess); /* increments sess refcount */ assert(ret==1); SSL_SESSION_free(sess); } } ssl = SSL_new(sslctx); SSL_CTX_free(sslctx); /* SSL_new() increments refcount */ if (!ssl) { TFE_LOG_ERROR(mgr->logger, "ssl new failed."); return -1; } SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream); if (chello->sni) { SSL_set_tlsext_host_name(ssl, chello->sni); } if (chello->alpn && s_stream->up_parts.apln_enabled) { ret=SSL_set_alpn_protos(ssl, (unsigned char*)chello->alpn, strlen(chello->alpn)); assert(ret==0); } /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); s_stream->ssl=ssl; return 0; } void ssl_connect_server_ctx_free(struct ssl_connect_server_ctx * ctx) { if (ctx->s_stream != NULL) { ssl_stream_free(ctx->s_stream); } if (ctx->bev != NULL) { bufferevent_free(ctx->bev); ctx->bev = NULL; } if (ctx->f_peek_chello != NULL) { future_destroy(ctx->f_peek_chello); ctx->f_peek_chello = NULL; } free(ctx); return; } void wrap_ssl_connect_server_ctx_free(void *p) { struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *)p; ssl_connect_server_ctx_free(ctx); } enum ssl_stream_action ssl_upstream_create_result_release_action(future_result_t * result) { struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result; return ctx->s_stream->up_parts.action; } evutil_socket_t ssl_upstream_create_result_release_fd(future_result_t * result) { struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result; return ctx->fd_upstream; } struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result) { struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result; struct ssl_stream * ret = ctx->s_stream; ctx->s_stream = NULL; //giveup ownership return ret; } struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result) { struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result; struct bufferevent * ret = ctx->bev; ctx->bev = NULL; //giveup ownership return ret; } const char* ssl_stream_dump_info(struct ssl_stream *stream, char* buffer, size_t sz) { snprintf(buffer, sz, "%s:%s", SSL_get_version(stream->ssl), stream->dir==CONN_DIR_UPSTREAM ? stream->up_parts.client_hello->sni:NULL); return buffer; } void ssl_stream_set_cmsg_string(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, const char* value_str) { struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream); UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)value_str, (uint16_t)strlen(value_str)); assert(ret==0); return; } static void ssl_stream_set_cmsg_uint8(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, uint8_t value_int) { struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream); UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)&value_int, (uint16_t)sizeof(value_int)); assert(ret==0); return; } static void ssl_stream_set_cmsg_integer(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, uint64_t value_int) { struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream); UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)&value_int, (uint16_t)sizeof(value_int)); assert(ret==0); return; } unsigned long ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struct ssl_mgr* mgr) { unsigned long sslerr=0, ret_sslerr=0; int fd=bufferevent_getfd(bev); char* addr_string=tfe_string_addr_create_by_fd(fd, dir); void* logger=mgr->logger; int fs_id=-1; /* Can happen for socket errs, ssl errs; * may happen for unclean ssl socket shutdowns. */ sslerr = bufferevent_get_openssl_error(bev); ret_sslerr=ERR_GET_REASON(sslerr); switch(ret_sslerr) { case SSL_R_INAPPROPRIATE_FALLBACK: if(dir==CONN_DIR_DOWNSTREAM) fs_id=SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK; break; case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: if(dir==CONN_DIR_DOWNSTREAM) fs_id=SSL_DOWN_ERR_NO_CERT; break; case SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM: case SSL_R_UNSUPPORTED_PROTOCOL: case SSL_R_UNSUPPORTED_SSL_VERSION: case SSL_R_UNSUPPORTED_STATUS_TYPE: case SSL_R_NO_PROTOCOLS_AVAILABLE: if(dir==CONN_DIR_UPSTREAM) fs_id=SSL_UP_ERR_UNSUPPORT_PROTO; case SSL_R_NO_CIPHERS_AVAILABLE: if(dir==CONN_DIR_UPSTREAM) fs_id=SSL_UP_ERR_NO_CIPHER; break; default: fs_id=-1; } if(fs_id>=0) { mgr->stat_val[fs_id]++; } if (!errno && !sslerr) { /* We have disabled notification for unclean shutdowns * so this should not happen; log a warning. */ TFE_LOG_ERROR(logger,"Warning: Spurious error from " "bufferevent (errno=0, sslerr=0)\n"); } else if (ERR_GET_REASON(sslerr) == SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE) { /* these can happen due to client cert auth, * only log error if debugging is activated */ TFE_LOG_ERROR(logger,"Handshake Error from bufferevent of ssl %s %s: " "%i:%s %lu:%i:%s:%i:%s:%i:%s", tfe_stream_conn_dir_to_str(dir), addr_string, errno, errno ? strerror(errno) : "-", sslerr, ERR_GET_REASON(sslerr), sslerr ? ERR_reason_error_string(sslerr) : "-", ERR_GET_LIB(sslerr), sslerr ? ERR_lib_error_string(sslerr) : "-", ERR_GET_FUNC(sslerr), sslerr ? ERR_func_error_string(sslerr) : "-"); /* after log, reset errno */ errno = 0; while ((sslerr = bufferevent_get_openssl_error(bev))) { TFE_LOG_ERROR(logger,"Additional SSL error: " "%lu:%i:%s:%i:%s:%i:%s", sslerr, ERR_GET_REASON(sslerr), ERR_reason_error_string(sslerr), ERR_GET_LIB(sslerr), ERR_lib_error_string(sslerr), ERR_GET_FUNC(sslerr), ERR_func_error_string(sslerr)); } } else { /* real errors */ TFE_LOG_ERROR(logger,"Error from bufferevent of ssl %s %s: " "%i:%s %lu:%i:%s:%i:%s:%i:%s", tfe_stream_conn_dir_to_str(dir), addr_string, errno, errno ? strerror(errno) : "-", sslerr, ERR_GET_REASON(sslerr), sslerr ? ERR_reason_error_string(sslerr) : "-", ERR_GET_LIB(sslerr), sslerr ? ERR_lib_error_string(sslerr) : "-", ERR_GET_FUNC(sslerr), sslerr ? ERR_func_error_string(sslerr) : "-"); /* after log, reset errno */ errno = 0; while ((sslerr = bufferevent_get_openssl_error(bev))) { TFE_LOG_ERROR(logger,"Additional SSL error: " "%lu:%i:%s:%i:%s:%i:%s\n", sslerr, ERR_GET_REASON(sslerr), ERR_reason_error_string(sslerr), ERR_GET_LIB(sslerr), ERR_lib_error_string(sslerr), ERR_GET_FUNC(sslerr), ERR_func_error_string(sslerr)); } } free(addr_string); return ret_sslerr; } void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr, struct ssl_mgr* mgr) { struct ssl_upstream_parts* s_upstream=NULL; assert(sslerr); switch(s_stream->dir) { case CONN_DIR_DOWNSTREAM: s_upstream= &(s_stream->peer->up_parts); if( (sslerr==SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN || sslerr==SSL_R_SSLV3_ALERT_BAD_CERTIFICATE || sslerr==SSL_R_TLSV1_ALERT_UNKNOWN_CA || sslerr==SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE || sslerr==SSL_R_UNKNOWN_CERTIFICATE_TYPE) && s_upstream->is_server_cert_verify_passed && s_upstream->verify_result.is_hostmatched) { s_upstream->svc_status.cli_pinning_status=PINNING_ST_PINNING; // This feature is also displayed when the root certificate is not installed. // Here no longer set the pinning state to cmsg, but use the pinng state // corrected by app_is_not_pinnig in the peek_chello_on_succ function // ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING); ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status); } else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN) { s_upstream->svc_status.has_protocol_errors=1; ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status); } break; case CONN_DIR_UPSTREAM: s_upstream=&(s_stream->up_parts); s_upstream->svc_status.has_protocol_errors=1; ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->up_parts.svc_status)); break; default: assert(0); } return; } void ssl_stream_process_zero_eof(struct ssl_stream * s_stream, struct ssl_mgr* mgr) { struct ssl_upstream_parts* s_upstream=NULL; if(s_stream->dir==CONN_DIR_UPSTREAM) { return; } assert(mgr==s_stream->mgr); s_upstream=&s_stream->peer->up_parts; if(s_upstream->verify_result.is_hostmatched && s_upstream->is_server_cert_verify_passed ) { const char *sni = s_upstream->client_hello ? (s_upstream->client_hello->sni ? s_upstream->client_hello->sni : "null"): "null"; TFE_LOG_DEBUG(mgr->logger, "sni:%s cert verify passed and hit zero eof, set protocol errors", sni); s_upstream->svc_status.has_protocol_errors=1; ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->peer->up_parts.svc_status)); } s_stream->error=SSL_STREAM_R_CLIENT_CLOSED; return; } /* * Callback for meta events on the up- and downstream connection bufferevents. * Called when EOF has been reached, a connection has been made, and on errors. */ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, void * arg) { struct promise * p = (struct promise *) arg; struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p); struct ssl_stream * s_stream = ctx->s_stream; assert(s_stream->dir==CONN_DIR_UPSTREAM); struct ssl_upstream_parts* s_upstream=&(s_stream->up_parts); struct ssl_mgr* mgr=s_stream->mgr; SSL_SESSION * ssl_sess = NULL; char error_str[TFE_STRING_MAX]; uint64_t jiffies_ms; unsigned long sslerr=0; if (events & BEV_EVENT_ERROR) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); sslerr=ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, ctx->mgr); if(sslerr) { ssl_stream_process_error(s_stream, sslerr, mgr); } s_stream->error=SSL_STREAM_R_SERVER_PROTOCOL_ERROR; } else if(events & BEV_EVENT_EOF) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); s_stream->error=SSL_STREAM_R_SERVER_CLOSED; } else if(events & BEV_EVENT_TIMEOUT) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); s_stream->error=SSL_STREAM_R_CONNECT_SERVER_TIMEOUT; } else if(events & BEV_EVENT_CONNECTED) { bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success clock_gettime(CLOCK_MONOTONIC, &(ctx->end)); jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000; if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS) { TFE_LOG_INFO(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s", jiffies_ms, s_stream->tcp_stream->str_stream_info, s_upstream->client_hello->sni); } s_stream->connect_latency_ms=jiffies_ms; ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_SERVER_SIDE_LATENCY, jiffies_ms); if(!SSL_session_reused(s_stream->ssl)) { if(mgr->no_cert_verify) { s_upstream->is_server_cert_verify_passed=1; } else { s_upstream->is_server_cert_verify_passed = !!ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store, s_stream->ssl, s_stream->up_parts.client_hello->sni, &(s_stream->up_parts.verify_param), error_str, sizeof(error_str), &(s_stream->up_parts.verify_result)); TFE_LOG_DEBUG(g_default_logger, "addr:%s, sni:%s, is_cert_verify_passed:%d, cet_real_untrust:%d, verify_host_fail:%d, verify_issure_fail:%d, verify_self_signed_fail:%d, verify_expiry_date_fail:%d, verify_other_fail:%d, %s", s_stream->tcp_stream->str_stream_info, s_upstream->client_hello->sni, s_upstream->is_server_cert_verify_passed, ((s_upstream->verify_param.real_untrust & 0xff) ? 1 : 0), ((s_upstream->verify_param.real_untrust & 0x01) ? 1 : 0), ((s_upstream->verify_param.real_untrust & 0x02) ? 1 : 0), ((s_upstream->verify_param.real_untrust & 0x04) ? 1 : 0), ((s_upstream->verify_param.real_untrust & 0x08) ? 1 : 0), ((s_upstream->verify_param.real_untrust & 0x10) ? 1 : 0), (s_upstream->is_server_cert_verify_passed == 0 ? error_str : "")); s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct; s_upstream->svc_status.is_ev=s_upstream->verify_result.is_ev; ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &(s_upstream->svc_status)); TFE_LOG_DEBUG(mgr->logger, "stream:%s sni:%s hostmatch:%d, ct:%d, ev:%d", s_stream->tcp_stream->str_stream_info, s_upstream->client_hello->sni, s_upstream->verify_result.is_hostmatched, s_upstream->verify_result.is_ct, s_upstream->verify_result.is_ev); if((!s_upstream->is_server_cert_verify_passed || !s_upstream->verify_result.is_hostmatched) && s_upstream->block_fake_cert) { s_stream->up_parts.action=SSL_ACTION_SHUTDOWN; } } if(s_upstream->is_server_cert_verify_passed) { if(!mgr->no_sesscache && s_stream->up_parts.action==SSL_ACTION_INTERCEPT) { //ONLY verified session is cacheable. //The reference count of the SSL_SESSION is not incremented, so no need to free. ssl_sess = SSL_get0_session(s_stream->ssl); up_session_set(mgr->up_sess_cache, (struct sockaddr *)&(ctx->addr), ctx->addrlen, s_upstream->client_hello->sni, SSL_version(s_stream->ssl), ssl_sess); } } else { ATOMIC_INC(&(mgr->stat_val[SSL_FAKE_CRT])); char* addr_str=tfe_string_addr_create_by_fd(ctx->fd_upstream, CONN_DIR_UPSTREAM); TFE_LOG_INFO(mgr->logger, "Fake Cert %s %s : %s", addr_str, s_upstream->client_hello->sni, error_str); free(addr_str); } } else { //Do not perform cert verification on reused session. s_upstream->is_server_cert_verify_passed=1; } if(mgr->log_master_key) { log_ssl_master_key(s_stream, CONN_DIR_UPSTREAM, mgr->fp_master_key); } const unsigned char *alpn_data=NULL; unsigned int alpn_len=0; SSL_get0_alpn_selected(s_stream->ssl, &alpn_data, &alpn_len); if(alpn_len>0&&alpn_data!=NULL) { if(0==memcmp(alpn_data, SSL_ALPN_HTTP_1_1+1, alpn_len)) { s_stream->alpn_selected=SSL_ALPN_HTTP_1_1; } else if(0==memcmp(alpn_data, SSL_ALPN_HTTP_2+1, alpn_len)) { s_stream->alpn_selected=SSL_ALPN_HTTP_2; } else { s_stream->alpn_selected=NULL; } } s_stream->negotiated_version=SSL_version(s_stream->ssl); ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_SERVER_SIDE_VERSION, SSL_get_version(s_stream->ssl)); ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_CERT_VERIFY, s_upstream->is_server_cert_verify_passed); promise_success(p, ctx); } if(s_stream->error) { ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error)); snprintf(error_str, sizeof(error_str), "%s, sni=%s", ssl_stream_get_error_string(s_stream->error), s_upstream->client_hello->sni); promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str); } wrap_ssl_connect_server_ctx_free(ctx); return; } static void peek_chello_on_succ(future_result_t * result, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_server_ctx* ctx = (struct ssl_connect_server_ctx *) promise_get_ctx(p); struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id); struct ssl_stream* s_stream=NULL; struct ssl_chello* chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. if(chello->sni==NULL) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_SNI])); } int ret=0; struct ssl_service_status* svc_status=NULL; clock_gettime(CLOCK_MONOTONIC, &(ctx->start)); s_stream= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL, NULL, ctx->tcp_stream); svc_status=&s_stream->up_parts.svc_status; ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, s_stream->tcp_stream, svc_status); if(ret==1) { TFE_LOG_DEBUG(ctx->mgr->logger, "%s %s service status pinning:%d, mauth:%d, err:%d, ct:%d, ev:%d, ja3_pinning_status:%d", s_stream->tcp_stream->str_stream_info, chello->sni, svc_status->cli_pinning_status, svc_status->is_mutual_auth, svc_status->has_protocol_errors, svc_status->is_ct, svc_status->is_ev, svc_status->ja3_pinning_status); } switch (svc_status->ja3_pinning_status) { case JA3_PINNING_STATUS_NOT_PINNING: ctx->mgr->svc_cache->stat.app_not_pinning_cnt++; ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_NOT_PINNING); break; case JA3_PINNING_STATUS_IS_PINNING: ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING); break; case JA3_PINNING_STATUS_UNKNOWN: /* fall through */ default: ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, svc_status->cli_pinning_status); break; } if(ctx->mgr->on_new_upstream_cb) { s_stream->up_parts.action=ctx->mgr->on_new_upstream_cb(s_stream, ctx->mgr->upstream_cb_param); } else { s_stream->up_parts.action=SSL_ACTION_INTERCEPT; } if (ATOMIC_READ(&certstore_is_unavailable) > 3) { s_stream->up_parts.action=SSL_ACTION_PASSTHROUGH; ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_PASSTHROUGH_REASON, "Certstore Unavailable"); TFE_LOG_ERROR(ctx->mgr->logger, "CertStore is unavailable, PASSTHROUGH"); } ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_INTERCEPT_STATE, s_stream->up_parts.action); ctx->s_stream = s_stream; if(s_stream->up_parts.action==SSL_ACTION_PASSTHROUGH) { promise_dettach_ctx(p); promise_success(p, ctx); wrap_ssl_connect_server_ctx_free(ctx); } else { if (upstream_ossl_init(s_stream)) { promise_dettach_ctx(p); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream ossl init failed"); wrap_ssl_connect_server_ctx_free(ctx); return; } ctx->bev = bufferevent_openssl_socket_new(evbase, ctx->fd_upstream, ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE ); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1); bufferevent_setcb(ctx->bev, NULL, NULL, ssl_server_connected_eventcb, p); bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only future_destroy(ctx->f_peek_chello); ctx->f_peek_chello = NULL; } return; } static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p); ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO])); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream."); wrap_ssl_connect_server_ctx_free(ctx); return; } void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream) { struct promise * p = future_to_promise(f); struct ssl_connect_server_ctx * ctx = ALLOC(struct ssl_connect_server_ctx, 1); int ret = 0; ctx->addrlen = sizeof(ctx->addr); ret = getpeername(fd_upstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen)); if(ret!=0) { ssl_connect_server_ctx_free(ctx); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream fd closed"); return; } struct event_base* evbase=tfe_proxy_get_work_thread_evbase(tcp_stream->thread_id); ctx->fd_downstream = fd_downstream; ctx->fd_upstream = fd_upstream; ctx->tcp_stream = tcp_stream; ctx->mgr = mgr; promise_set_ctx(p, ctx, wrap_ssl_connect_server_ctx_free); ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p); ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); } static int ossl_session_ticket_key_callback(SSL *ssl_conn, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc) { enum STEK_GET_RET ret=STEK_NOT_FOUND; const EVP_MD *digest=EVP_sha256(); const EVP_CIPHER *cipher=EVP_aes_256_cbc(); size_t size=32; struct ssl_stream* s_stream = (struct ssl_stream*) SSL_get_ex_data(ssl_conn, SSL_EX_DATA_IDX_SSLSTREAM); struct ssl_mgr* mgr= s_stream->mgr; void* logger = s_stream->mgr->logger; const char * sni = s_stream->peer->up_parts.client_hello->sni; struct sess_ticket_box * stek_box = s_stream->mgr->down_stek_box; struct sess_ticket_key cur_stek; unsigned char buf[33]={0}; ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY])); if (enc == 1) { sess_ticket_box_get_key_for_enc(stek_box, sni, &cur_stek); /* encrypt session stek */ if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1) { TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed"); ret=STEK_ERROR; goto leave; } if (EVP_EncryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1) { TFE_LOG_ERROR(mgr->logger, "EVP_EncryptInit_ex() failed"); ret=STEK_ERROR; goto leave; } if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1) { TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed"); ret=STEK_ERROR; goto leave; } memcpy(name, cur_stek.name, sizeof(cur_stek.name)); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW])); ret=STEK_FOUND_FRESH; } else { /* decrypt session stek */ ret=sess_ticket_box_get_key_for_dec(stek_box, sni, name, &cur_stek); if(ret==STEK_FOUND_FRESH||ret==STEK_FOUND_STALED) { if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1) { TFE_LOG_ERROR(logger, "HMAC_Init_ex() failed"); ret= STEK_ERROR; goto leave; } if (EVP_DecryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1) { TFE_LOG_ERROR(logger, "EVP_DecryptInit_ex() failed"); ret= STEK_ERROR; goto leave; } ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_REUSE])); } else { // full handshake and update the session stek_box TFE_LOG_INFO(logger, "ssl session stek_box decrypt, key: \"%*s\" not found",(int)(tfe_hexdump(buf, name ,16)-buf), buf); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND])); } } leave: return (int)ret; } /* * Called by OpenSSL when a new src SSL session is created. * OpenSSL increments the refcount before calling the callback and will * decrement it again if we return 0. Returning 1 will make OpenSSL skip * the refcount decrementing. In other words, return 0 if we did not * keep a pointer to the object (which we never do here). */ static int ossl_downsess_new_cb(SSL * ssl, SSL_SESSION * sess) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_CTX_EX_DATA_IDX_SSLMGR); #ifdef HAVE_SSLV2 /* Session resumption seems to fail for SSLv2 with protocol * parsing errors, so we disable caching for SSLv2. */ if (SSL_version(ssl) == SSL2_VERSION) { return 0; } #endif /* HAVE_SSLV2 */ if (sess && !mgr->no_sesscache) { down_session_set(mgr->down_sess_cache, sess); } return 0; } /* * Called by OpenSSL when a src SSL session should be removed. * OpenSSL calls SSL_SESSION_free() after calling the callback; * we do not need to free the reference here. */ static void ossl_downsess_remove_cb(SSL_CTX * sslctx, SSL_SESSION * sess) { struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_CTX_get_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR); assert(mgr != NULL); if (sess && !mgr->no_sesscache) { down_session_del(mgr->down_sess_cache, sess); } return; } /* * Called by OpenSSL when a src SSL session is requested by the client. OPENSSL_VERSION_NUMBER >= 0x10100000L required. */ static SSL_SESSION * ossl_downsess_get_cb(SSL * ssl, const unsigned char * id, int idlen, int * copy) { struct ssl_stream* s_stream = (struct ssl_stream*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM); struct ssl_mgr * mgr = s_stream->mgr; SSL_SESSION * sess=NULL; if(!mgr->no_sesscache) { *copy = 0; /* SSL should not increment reference count of session */ sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache, id, idlen); } return sess; } /* * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX. */ static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr) { SSL_CTX_set_options(sslctx, SSL_OP_ALL); SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG); SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); if (mgr->no_ssl2) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); } if (mgr->no_ssl3) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); } if (mgr->no_tls10) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1); } if (mgr->no_tls11) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1); } if (mgr->no_tls12) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2); } if(mgr->no_sessticket) { SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET); } if (!mgr->sslcomp) { SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION); } } static int alpn_select_proto_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg) { int rv; const unsigned char* selected_alpn=(const unsigned char*)arg; (void)ssl; unsigned char* _out=NULL; unsigned char _outlen=0; rv=SSL_select_next_proto(&_out, &_outlen, in, inlen, selected_alpn, strlen((const char*)selected_alpn)); if(rv==OPENSSL_NPN_NEGOTIATED) { *out=_out; *outlen=_outlen; return SSL_TLSEXT_ERR_OK; } else //rv==OPENSSL_NPN_NO_OVERLAP { return SSL_TLSEXT_ERR_NOACK; } } /* * Create and set up a new SSL_CTX instance for terminating SSL. * Set up all the necessary callbacks, the keyring, the keyring chain and key. */ void downstream_ossl_init(struct ssl_stream *s_stream) { struct ssl_mgr * mgr=s_stream->mgr; SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod()); if (!sslctx) return; struct keyring * crt=s_stream->down_parts.keyring; int negotiated_version=s_stream->peer->negotiated_version; const unsigned char* selected_alpn=s_stream->peer->alpn_selected; SSL * ssl = NULL; UNUSED int ret = 0; sslctx_set_opts(sslctx, mgr); SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers); //TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L if(negotiated_version) { if (SSL_CTX_set_min_proto_version(sslctx, negotiated_version) == 0 || SSL_CTX_set_max_proto_version(sslctx, negotiated_version) == 0) { SSL_CTX_free(sslctx); return; } } else if(mgr->ssl_min_version) { if (SSL_CTX_set_min_proto_version(sslctx, mgr->ssl_min_version) == 0 || SSL_CTX_set_max_proto_version(sslctx, mgr->ssl_max_version) == 0) { SSL_CTX_free(sslctx); return; } } SSL_CTX_sess_set_new_cb(sslctx, ossl_downsess_new_cb); SSL_CTX_sess_set_remove_cb(sslctx, ossl_downsess_remove_cb); SSL_CTX_sess_set_get_cb(sslctx, ossl_downsess_get_cb); if(!mgr->no_sessticket&&s_stream->peer->up_parts.client_hello->sni) { SSL_CTX_set_tlsext_ticket_key_cb(sslctx, ossl_session_ticket_key_callback); } SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); SSL_CTX_set_session_id_context(sslctx, (const unsigned char *) mgr->ssl_session_context, sizeof(mgr->ssl_session_context)); ret = SSL_CTX_set_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR, mgr);//used by down session cache get assert(ret == 1); if (mgr->dh) { SSL_CTX_set_tmp_dh(sslctx, mgr->dh); } else { SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback); } if (mgr->ecdhcurve) { EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } else { EC_KEY * ecdh = ssl_ec_by_name(NULL); SSL_CTX_set_tmp_ecdh(sslctx, ecdh); EC_KEY_free(ecdh); } if(s_stream->peer->up_parts.apln_enabled && selected_alpn) { SSL_CTX_set_alpn_select_cb(sslctx, alpn_select_proto_cb, (void*)selected_alpn); } SSL_CTX_use_certificate(sslctx, crt->cert); SSL_CTX_use_PrivateKey(sslctx, crt->key); for (int i = 0; i < sk_X509_num(crt->chain); i++) { X509 * c = sk_X509_value(crt->chain, i); ssl_x509_refcount_inc(c); /* next call consumes a reference */ SSL_CTX_add_extra_chain_cert(sslctx, c); } ssl = SSL_new(sslctx); SSL_CTX_free(sslctx); // SSL_new() increments refcount sslctx = NULL; ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream); assert(ret == 1); if (mgr->ssl_mode_release_buffers == 1) { /* lower memory footprint for idle connections */ SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS); } s_stream->ssl=ssl; return; } void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx) { X509_free(ctx->origin_crt); ctx->origin_crt=NULL; if (ctx->f_ask_keyring != NULL) { future_destroy(ctx->f_ask_keyring); ctx->f_ask_keyring = NULL; } //on success, bev_down and downstream has been transfered to caller by release** if (ctx->bev_down != NULL) { bufferevent_free(ctx->bev_down); } if (ctx->downstream != NULL) { ssl_stream_free(ctx->downstream); } free(ctx); return; } void wrap_ssl_connect_client_ctx_free(void * p) { struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *)p; ssl_connect_client_ctx_free(ctx); } struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result) { struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result; struct ssl_stream * ret = ctx->downstream; ctx->downstream = NULL; return ret; } struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result) { struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result; struct bufferevent * ret = ctx->bev_down; ctx->bev_down = NULL; return ret; } static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, void * arg) { struct promise * p = (struct promise *) arg; struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p); struct ssl_stream * s_stream = ctx->downstream; struct ssl_upstream_parts* s_upstream= &(ctx->peer->up_parts); struct ssl_mgr* mgr=s_stream->mgr; char error_str[TFE_STRING_MAX]={0}; uint64_t jiffies_ms=0; unsigned long sslerr=0; if (events & BEV_EVENT_ERROR) { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); sslerr=ssl_stream_log_error(bev, CONN_DIR_DOWNSTREAM, mgr); if(sslerr) { ssl_stream_process_error(s_stream, sslerr, mgr); } s_stream->error=SSL_STREAM_R_CLIENT_PROTOCOL_ERROR; } else if(events & BEV_EVENT_EOF) { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); ssl_stream_process_zero_eof(s_stream, mgr); } else if(events & BEV_EVENT_TIMEOUT) { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); s_stream->error=SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT; } else if(events & BEV_EVENT_CONNECTED) { clock_gettime(CLOCK_MONOTONIC, &(ctx->end)); jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000; if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS) { TFE_LOG_INFO(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s", jiffies_ms, s_stream->tcp_stream->str_stream_info, s_upstream->client_hello->sni); } s_stream->connect_latency_ms=jiffies_ms; ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_CLIENT_SIDE_LATENCY, jiffies_ms); bufferevent_disable(ctx->bev_down, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev_down, NULL, NULL, NULL, NULL); //leave a clean bev for on_success if(mgr->log_master_key) { log_ssl_master_key(s_stream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key); } s_stream->negotiated_version=SSL_version(s_stream->ssl); ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_CLIENT_SIDE_VERSION, SSL_get_version(s_stream->ssl)); promise_success(p, ctx); } if(s_stream->error) { ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error)); snprintf(error_str, sizeof(error_str), "%s : sni=%s", ssl_stream_get_error_string(s_stream->error), s_upstream->client_hello->sni); promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str); } ssl_connect_client_ctx_free(ctx); return; } void ask_keyring_on_succ(void * result, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_get_ctx(p); struct keyring * kyr = NULL; struct ssl_mgr * mgr = ctx->ssl_mgr; struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id); kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing. clock_gettime(CLOCK_MONOTONIC, &(ctx->start)); ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, kyr, ctx->peer, ctx->tcp_stream); downstream_ossl_init(ctx->downstream); ctx->bev_down = bufferevent_openssl_socket_new(evbase, ctx->fd_downstream, ctx->downstream->ssl, BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p); bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only future_destroy(ctx->f_ask_keyring); ctx->f_ask_keyring = NULL; } void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p); promise_failed(p, error, what); ssl_connect_client_ctx_free(ctx); return; } /* * Create a SSL stream for the incoming connection, based on the upstream. */ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream) { assert(upstream->dir == CONN_DIR_UPSTREAM); const char* sni=NULL; struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1); ctx->ssl_mgr = mgr; ctx->fd_downstream = fd_downstream; ctx->tcp_stream = tcp_stream; struct event_base * evbase=tfe_proxy_get_work_thread_evbase(tcp_stream->thread_id); struct evdns_base* dnsbase=tfe_proxy_get_work_thread_dnsbase(tcp_stream->thread_id); struct evhttp_connection *evhttp=tfe_proxy_get_work_thread_evhttp(tcp_stream->thread_id); if (upstream != NULL) { ctx->peer = upstream; ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl); sni=upstream->up_parts.client_hello->sni; } struct promise * p = future_to_promise(f); promise_set_ctx(p, ctx, wrap_ssl_connect_client_ctx_free); ctx->f_ask_keyring = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p); ctx->is_origin_crt_verify_passed = upstream->up_parts.is_server_cert_verify_passed; int keyring_id = 0; if (ctx->is_origin_crt_verify_passed) { keyring_id = upstream->up_parts.keyring_for_trusted; } else { keyring_id = upstream->up_parts.keyring_for_untrusted; } key_keeper_async_ask(ctx->f_ask_keyring, mgr->key_keeper, sni, keyring_id, ctx->origin_crt, ctx->is_origin_crt_verify_passed, evbase, dnsbase, evhttp); return; } /* * Cleanly shutdown an SSL session on file descriptor fd using low-level * file descriptor readiness events on event base evbase. * Guarantees that SSL and the corresponding SSL_CTX are freed and the * socket is closed, eventually, or in the case of fatal errors, immediately. */ void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev) { UNUSED struct ssl_shutdown_ctx * sslshutctx = NULL; evutil_socket_t fd=-1; fd=bufferevent_getfd(bev); assert(fd==s_stream->_do_not_use.fd); unsigned long sslerr=0; if (s_stream->dir == CONN_DIR_UPSTREAM) { size_t rx_offset_this_time = 0; tfe_stream_info_get(s_stream->tcp_stream, INFO_FROM_UPSTREAM_RX_OFFSET, &rx_offset_this_time, sizeof(rx_offset_this_time)); const char * sni = (s_stream->up_parts.client_hello && s_stream->up_parts.client_hello->sni) ? s_stream->up_parts.client_hello->sni : "null"; TFE_LOG_DEBUG(g_default_logger, "ssl up stream close, rx_offset:%zu, sni:%s", rx_offset_this_time, sni); } if(errno) { sslerr=ssl_stream_log_error(bev, s_stream->dir, s_stream->mgr); if(sslerr) { ssl_stream_process_error(s_stream, sslerr, s_stream->mgr); } } //sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); //pxy_ssl_shutdown_cb(fd, 0, sslshutctx); bufferevent_setcb(bev, NULL, NULL, NULL, NULL); SSL_set_shutdown(s_stream->ssl, SSL_RECEIVED_SHUTDOWN); SSL_shutdown(s_stream->ssl); bufferevent_disable(bev, EV_READ|EV_WRITE); struct bufferevent *ubev = bufferevent_get_underlying(bev); if (ubev) { bufferevent_disable(ubev, EV_READ|EV_WRITE); bufferevent_setfd(ubev, -1); bufferevent_setcb(ubev, NULL, NULL, NULL, NULL); bufferevent_free(ubev); } ssl_stream_free(s_stream); } int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file) { int ret = ssl_trusted_cert_storage_add(mgr->trust_CA_store, SSL_X509_OBJ_CERT, pem_file); if (ret == 1) { ATOMIC_INC(&((*(mgr->svc_cache)).stat.trusted_cert_cnt)); } return ret; } int ssl_manager_del_trust_ca(struct ssl_mgr* mgr, const char* pem_file) { int ret = ssl_trusted_cert_storage_del(mgr->trust_CA_store, SSL_X509_OBJ_CERT, pem_file); if (ret == 1) { ATOMIC_DEC(&((*(mgr->svc_cache)).stat.trusted_cert_cnt)); } return ret; } int ssl_manager_add_crl(struct ssl_mgr* mgr, const char* pem_file) { return ssl_trusted_cert_storage_add(mgr->trust_CA_store, SSL_X509_OBJ_CRL, pem_file); } int ssl_manager_del_crl(struct ssl_mgr* mgr, const char* pem_file) { return ssl_trusted_cert_storage_del(mgr->trust_CA_store, SSL_X509_OBJ_CRL, pem_file); } void ssl_manager_reset_trust_ca(struct ssl_mgr* mgr) { ATOMIC_ZERO(&((*(mgr->svc_cache)).stat.trusted_cert_cnt)); ssl_trusted_cert_storage_reset(mgr->trust_CA_store); return; } void ssl_manager_reset_trust_ca_finish(struct ssl_mgr* mgr) { ssl_trusted_cert_storage_reset_finish(mgr->trust_CA_store); return; } int ssl_stream_set_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, int opt_val) { struct cert_verify_param *verify_param=&(upstream->up_parts.verify_param); assert(upstream->dir==CONN_DIR_UPSTREAM); switch(opt_type) { case SSL_STREAM_OPT_NO_VERIFY_SELF_SIGNED: verify_param->no_verify_self_signed=opt_val; break; case SSL_STREAM_OPT_NO_VERIFY_COMMON_NAME: verify_param->no_verify_cn=opt_val; break; case SSL_STREAM_OPT_NO_VERIFY_ISSUER: verify_param->no_verify_issuer=opt_val; break; case SSL_STREAM_OPT_NO_VERIFY_EXPIRY_DATE: verify_param->no_verify_expiry_date=opt_val; break; case SSL_STREAM_OPT_BLOCK_FAKE_CERT: upstream->up_parts.block_fake_cert=opt_val; break; case SSL_STREAM_OPT_PROTOCOL_MIN_VERSION: upstream->ssl_min_version=opt_val; break; case SSL_STREAM_OPT_PROTOCOL_MAX_VERSION: upstream->ssl_max_version=opt_val; break; case SSL_STREAM_OPT_ENABLE_ALPN: upstream->up_parts.apln_enabled=opt_val; break; case SSL_STREAM_OPT_KEYRING_FOR_TRUSTED: upstream->up_parts.keyring_for_trusted=opt_val; break; case SSL_STREAM_OPT_KEYRING_FOR_UNTRUSTED: upstream->up_parts.keyring_for_untrusted=opt_val; break; default: assert(0); return 0; } return 1; } int ssl_stream_get_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, int *opt_val) { struct ssl_service_status* svc=&upstream->up_parts.svc_status; UNUSED int ret=0; switch(opt_type) { case SSL_STREAM_OPT_IS_EV_CERT: *opt_val=svc->is_ev; break; case SSL_STREAM_OPT_IS_CT_CERT: *opt_val=svc->is_ct; break; case SSL_STREAM_OPT_IS_MUTUAL_AUTH: *opt_val=svc->is_mutual_auth; break; case SSL_STREAM_OPT_PINNING_STATUS: *opt_val=svc->cli_pinning_status; break; case SSL_STREAM_OPT_JA3_PINNING_STATUS: *opt_val=svc->ja3_pinning_status; break; case SSL_STREAM_OPT_HAS_PROTOCOL_ERRORS: *opt_val=svc->has_protocol_errors; break; default: return -1; } return 0; } uint64_t ssl_stream_get_policy_id(struct ssl_stream *upstream) { uint16_t out_size; uint64_t policy_id = 0; struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream); int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_POLICY_ID, (unsigned char *)&policy_id, sizeof(policy_id), &out_size); assert(ret == 0); return policy_id; } int ssl_stream_get_decrypted_profile_id(struct ssl_stream *upstream) { uint16_t out_size; int profile_id = 0; struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream); int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DECRYPTION_PROFILE_ID, (unsigned char *)&profile_id, sizeof(profile_id), &out_size); assert(ret == 0); return profile_id; } int ssl_stream_get_trusted_keyring_profile_id(struct ssl_stream *upstream) { uint16_t out_size; int keyring_id = 0; struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream); int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_KEYRING_FOR_TRUSTED_ID, (unsigned char *)&keyring_id, sizeof(keyring_id), &out_size); assert(ret == 0); return keyring_id; } int ssl_stream_get_untrusted_keyring_profile_id(struct ssl_stream *upstream) { uint16_t out_size; int keyring_id = 0; struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream); int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_KEYRING_FOR_UNTRUSTED, (unsigned char *)&keyring_id, sizeof(keyring_id), &out_size); assert(ret == 0); return keyring_id; } int ssl_stream_get_string_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, char* in_buff, size_t sz) { const char* sni=upstream->up_parts.client_hello->sni?upstream->up_parts.client_hello->sni:"null"; switch(opt_type) { case SSL_STREAM_OPT_SNI: strncpy(in_buff, sni, sz); break; case SSL_STREAM_OPT_ADDR: strncpy(in_buff, upstream->tcp_stream->str_stream_info, sz); break; default: assert(0); return -1; } return 0; }