diff --git a/ci/travis.sh b/ci/travis.sh index 93934d7..3acabdc 100644 --- a/ci/travis.sh +++ b/ci/travis.sh @@ -51,5 +51,5 @@ if [ -n "${PACKAGE}" ]; then fi if [ -n "${UPLOAD_SYMBOL_FILES}" ]; then - sentry-cli upload-dif -t elf build/ + sentry-cli upload-dif -t elf ./ fi diff --git a/platform/include/internal/ssl_stream_core.h b/platform/include/internal/ssl_stream_core.h index 11af223..399b13a 100644 --- a/platform/include/internal/ssl_stream_core.h +++ b/platform/include/internal/ssl_stream_core.h @@ -15,7 +15,7 @@ enum ssl_stream_error SSL_STREAM_R_CLIENT_PROTOCOL_ERROR, __SSL_STREAM_R_MAX }; - + struct ssl_mgr; @@ -28,8 +28,6 @@ void ssl_stream_process_zero_eof(struct ssl_stream * s_stream, struct ssl_mgr* m enum ssl_stream_action ssl_upstream_create_result_release_action(future_result_t * result); -evutil_socket_t ssl_upstream_create_result_release_fd(future_result_t * result); - struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result); struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result); void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, @@ -44,7 +42,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream); -void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev); +void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev); const char* ssl_stream_dump_info(struct ssl_stream *stream, char* buffer, size_t sz); //Follow functions are allowed to call during runtime. int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file); diff --git a/platform/src/acceptor_kni_v2.cpp b/platform/src/acceptor_kni_v2.cpp index cb4e1cc..4ec4ad3 100644 --- a/platform/src/acceptor_kni_v2.cpp +++ b/platform/src/acceptor_kni_v2.cpp @@ -47,6 +47,8 @@ void acceptor_kni_v2_event(evutil_socket_t fd, short what, void * user) struct tfe_cmsg * cmsg = NULL; int * __fds = NULL; + unsigned int __nr_fds = 0; + assert(__ctx != NULL && __ctx->thread == pthread_self()); assert(what & EV_READ); @@ -91,11 +93,19 @@ void acceptor_kni_v2_event(evutil_socket_t fd, short what, void * user) TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_FIRSTHDR() from incoming fds."); goto __drop_recieved_fds; } - - __fds = (int *) (CMSG_DATA(__cmsghdr)); - if (unlikely(__fds == NULL)) + + switch(__cmsghdr->cmsg_len) { - TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_DATA() from incoming fds."); + case CMSG_LEN(0 * sizeof(int)): { __nr_fds = 0; break;} + case CMSG_LEN(1 * sizeof(int)): { __nr_fds = 1; break;} + case CMSG_LEN(2 * sizeof(int)): { __nr_fds = 2; break; } + default: assert(0); + } + + __fds = (int *) (CMSG_DATA(__cmsghdr)); + if (unlikely(__fds == NULL || __nr_fds < 2)) + { + TFE_LOG_ERROR(__ctx->logger, "No available file descriptors, drop the incoming fds."); goto __drop_recieved_fds; } @@ -120,9 +130,13 @@ __die: return; __drop_recieved_fds: - TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL, 2); - if (__fds != NULL) evutil_closesocket(__fds[0]); - if (__fds != NULL) evutil_closesocket(__fds[1]); + TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL, __nr_fds); + for (unsigned int i = 0; i < __nr_fds; i++) + { + evutil_closesocket(__fds[i]); + } + + assert(__nr_fds <= 2); } void * acceptor_kni_v2_event_thread_entry(void * args) diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index c0aa60c..952ab9f 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -76,12 +76,12 @@ enum ssl_stream_stat SSL_UP_CACHE_SZ, SSL_UP_CACHE_QUERY, SSL_UP_CACHE_HIT, - + SSL_DOWN_NEW, SSL_DOWN_ERR, SSL_DOWN_ERR_NO_CERT, - SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK, - + SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK, + SSL_DOWN_CLOSING, SSL_DOWN_CLOSED, SSL_DOWN_DIRTY_CLOSED, @@ -92,7 +92,7 @@ enum ssl_stream_stat SSL_DOWN_TICKET_REUSE, SSL_DOWN_TICKET_NOTFOUND, SSL_DOWN_TIKCET_QUERY, - + SSL_NO_CHELLO, SSL_NO_SNI, SSL_FAKE_CRT, @@ -140,7 +140,7 @@ struct ssl_mgr struct sess_cache * down_sess_cache; struct sess_cache * up_sess_cache; struct sess_ticket_box * down_stek_box; - + struct ssl_service_cache* svc_cache; ssl_stream_new_hook* on_new_upstream_cb; void* upstream_cb_param; @@ -165,7 +165,7 @@ struct ssl_mgr unsigned int log_master_key; char master_key_file[TFE_PATH_MAX]; FILE* fp_master_key; - + void * logger; screen_stat_handle_t fs_handle; long long stat_val[SSL_STAT_MAX]; @@ -185,7 +185,7 @@ struct ssl_bypass_condition }; struct ssl_upstream_parts { - + struct cert_verify_param verify_param; struct cert_verify_result verify_result; char block_fake_cert; @@ -194,7 +194,7 @@ struct ssl_upstream_parts int apln_enabled; int keyring_id; struct ssl_chello * client_hello; - int is_server_cert_verify_passed; + int is_server_cert_verify_passed; }; struct ssl_downstream_parts { @@ -243,9 +243,9 @@ struct ssl_connect_server_ctx evutil_socket_t fd_upstream; evutil_socket_t fd_downstream; - + struct tfe_stream* tcp_stream; - + struct future * f_peek_chello; struct timespec start,end; }; @@ -343,7 +343,7 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) struct ssl_mgr *mgr=(struct ssl_mgr *)arg; int i=0; if(!mgr->no_sesscache) - { + { ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT])); ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT])); } @@ -352,14 +352,14 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times; mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue; mgr->stat_val[KEY_KEEPER_CACHE_SIZE]=keeper_stat.cached_num; - + struct ssl_service_cache_statistics svc_stat; memset(&svc_stat, 0, sizeof(svc_stat)); ssl_service_cache_stat(mgr->svc_cache, &svc_stat); mgr->stat_val[SSL_SVC_PINNING]=svc_stat.pinning_cli_cnt; mgr->stat_val[SSL_SVC_MAUTH]=svc_stat.mutual_auth_cli_cnt; mgr->stat_val[SSL_SVC_CT_CERT]=svc_stat.ct_srv_cnt; - mgr->stat_val[SSL_SVC_EV_CERT]=svc_stat.ev_srv_cnt; + mgr->stat_val[SSL_SVC_EV_CERT]=svc_stat.ev_srv_cnt; mgr->stat_val[SSL_SVC_APP_NOT_PINNING]=svc_stat.app_not_pinning_cnt; for(i=0;ino_sessticket) { spec[SSL_DOWN_TICKET_NEW]="dtkt_new"; @@ -417,7 +417,7 @@ void ssl_stat_init(struct ssl_mgr * mgr) spec[SSL_SVC_CT_CERT]="ssl_ct_crt"; spec[SSL_SVC_EV_CERT]="ssl_ev_crt"; spec[SSL_SVC_APP_NOT_PINNING]="app_no_pinning"; - + for(i=0;ifs_handle, ID_INVISBLE, &value, sizeof(value)); value=mgr->fs_id[SSL_UP_CACHE_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); - + FS_register_ratio(mgr->fs_handle, - mgr->fs_id[SSL_UP_CACHE_HIT], - mgr->fs_id[SSL_UP_CACHE_QUERY], + mgr->fs_id[SSL_UP_CACHE_HIT], + mgr->fs_id[SSL_UP_CACHE_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, @@ -444,10 +444,10 @@ void ssl_stat_init(struct ssl_mgr * mgr) FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); value=mgr->fs_id[SSL_DOWN_CACHE_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); - + FS_register_ratio(mgr->fs_handle, - mgr->fs_id[SSL_DOWN_CACHE_HIT], - mgr->fs_id[SSL_DOWN_CACHE_QUERY], + mgr->fs_id[SSL_DOWN_CACHE_HIT], + mgr->fs_id[SSL_DOWN_CACHE_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, @@ -457,21 +457,21 @@ void ssl_stat_init(struct ssl_mgr * mgr) { value=mgr->fs_id[SSL_DOWN_TIKCET_QUERY]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); - + value=mgr->fs_id[SSL_DOWN_TICKET_REUSE]; FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); - + FS_register_ratio(mgr->fs_handle, - mgr->fs_id[SSL_DOWN_TICKET_REUSE], - mgr->fs_id[SSL_DOWN_TIKCET_QUERY], + mgr->fs_id[SSL_DOWN_TICKET_REUSE], + mgr->fs_id[SSL_DOWN_TIKCET_QUERY], 1, FS_STYLE_STATUS, FS_CALC_CURRENT, "dtkt_hit"); } - + struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here. - mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr); + mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr); evtimer_add(mgr->gcev, &gc_delay); return; @@ -537,12 +537,12 @@ static void ssl_stream_free(struct ssl_stream * s_stream) { key_keeper_free_keyring(s_stream->down_parts.keyring); s_stream->down_parts.keyring = NULL; - } + } ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED])); break; case CONN_DIR_UPSTREAM: if (s_stream->up_parts.client_hello != NULL) - { + { ssl_chello_free(s_stream->up_parts.client_hello); s_stream->up_parts.client_hello = NULL; } @@ -609,7 +609,7 @@ void ssl_manager_destroy(struct ssl_mgr * mgr) } -struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, +struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, struct event_base * ev_base_gc, void * logger) { unsigned int stek_group_num = 0; @@ -656,7 +656,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section MESA_load_profile_uint_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0); - MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers, + MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers, sizeof(mgr->default_ciphers), DFLT_CIPHERS); MESA_load_profile_uint_def(ini_profile, section, "no_session_cache", &(mgr->no_sesscache), 0); @@ -670,14 +670,14 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section &(mgr->sess_cache_slots), 4 * 1024 * 1024); MESA_load_profile_uint_def(ini_profile, section, "session_cache_expire_seconds", &(mgr->sess_expire_seconds), 30 * 60); - + if(!mgr->no_sesscache) { mgr->up_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); mgr->down_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); } - + MESA_load_profile_uint_def(ini_profile, section, "stek_group_num", &stek_group_num, 1); MESA_load_profile_uint_def(ini_profile, section, "stek_rotation_time", &stek_rotation_time, 3600); @@ -712,7 +712,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section TFE_LOG_ERROR(logger, "Certificate Manager initiate failed."); goto error_out; } - + MESA_load_profile_uint_def(ini_profile, section, "trusted_cert_load_local", &(mgr->trusted_cert_load_local), 1); @@ -721,10 +721,10 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section { MESA_load_profile_string_def(ini_profile, section, "trusted_cert_file", mgr->trusted_cert_file, sizeof(mgr->trusted_cert_file), "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"); - + MESA_load_profile_string_def(ini_profile, section, "trusted_cert_dir", mgr->trusted_cert_dir, sizeof(mgr->trusted_cert_dir), "./resource/tfe/trusted_storage"); - } + } mgr->trust_CA_store = ssl_trusted_cert_storage_create(mgr->trusted_cert_file, mgr->trusted_cert_dir, &(mgr->cert_verify_param)); if (mgr->trust_CA_store == NULL) { @@ -746,7 +746,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section mgr->log_master_key=0; } } - + mgr->fs_handle=tfe_proxy_get_fs_handle(); ssl_stat_init(mgr); return mgr; @@ -786,7 +786,7 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) //use promise_get_ctx instead of promise_dettach_ctx for try more times. struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *) promise_get_ctx(promise); enum chello_parse_result chello_status=CHELLO_PARSE_INVALID_FORMAT; - struct ssl_chello* chello=NULL; + struct ssl_chello* chello=NULL; const char * reason = NULL; unsigned char buf[2048]; ssize_t n = 0; @@ -826,7 +826,7 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) reason = "too many peek retries"; goto failed; } - + /* ssl_tls_clienthello_parse indicates that we * should retry later when we have more data, and we * haven't reached the maximum retry count yet. @@ -836,7 +836,7 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) * never actually read them, fd is still ready for * reading now. We use 25 * 0.2 s = 5 s timeout. */ struct timeval retry_delay = {0, 100}; - event_del(ctx->ev); + event_del(ctx->ev); event_free(ctx->ev); ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); assert(ctx->ev != NULL); @@ -942,10 +942,10 @@ static int upstream_ossl_init(struct ssl_stream* s_stream) TFE_LOG_ERROR(mgr->logger, "ssl set max proto version failed %d.", s_stream->ssl_max_version); return -1; } - + SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); SSL_CTX_set_client_cert_cb(sslctx, ossl_client_cert_cb); - + if(mgr->no_sesscache) { SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_OFF); @@ -955,7 +955,7 @@ static int upstream_ossl_init(struct ssl_stream* s_stream) SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_NO_INTERNAL); /* session resuming based on remote endpoint address and port */ sess = up_session_get(mgr->up_sess_cache, - (struct sockaddr *) &(s_stream->peer_addr), s_stream->peer_addrlen, chello->sni, + (struct sockaddr *) &(s_stream->peer_addr), s_stream->peer_addrlen, chello->sni, s_stream->ssl_min_version, s_stream->ssl_max_version); if (sess) { @@ -1098,7 +1098,7 @@ unsigned long ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir d } if(fs_id>=0) { - mgr->stat_val[fs_id]++; + mgr->stat_val[fs_id]++; } if (!errno && !sslerr) { @@ -1194,7 +1194,7 @@ void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr { s_upstream->svc_status.cli_pinning_status=PINNING_ST_PINNING; ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING); - ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &s_upstream->svc_status); + ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &s_upstream->svc_status); } else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN) { @@ -1204,7 +1204,7 @@ void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr break; case CONN_DIR_UPSTREAM: s_upstream=&(s_stream->up_parts); - s_upstream->svc_status.has_protocol_errors=1; + s_upstream->svc_status.has_protocol_errors=1; ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream->addr, &(s_stream->up_parts.svc_status)); break; default: @@ -1261,8 +1261,8 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, s_stream->error=SSL_STREAM_R_SERVER_PROTOCOL_ERROR; } else if(events & BEV_EVENT_EOF) - { - ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); + { + ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); s_stream->error=SSL_STREAM_R_SERVER_CLOSED; } else if(events & BEV_EVENT_TIMEOUT) @@ -1271,7 +1271,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, s_stream->error=SSL_STREAM_R_CONNECT_SERVER_TIMEOUT; } else if(events & BEV_EVENT_CONNECTED) - { + { bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success clock_gettime(CLOCK_MONOTONIC, &(ctx->end)); @@ -1279,7 +1279,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS) { TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s", - jiffies_ms, + jiffies_ms, s_stream->tcp_stream->str_stream_info, s_upstream->client_hello->sni); } @@ -1293,8 +1293,8 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, s_upstream->is_server_cert_verify_passed=1; } else - { - s_upstream->is_server_cert_verify_passed = ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store, + { + s_upstream->is_server_cert_verify_passed = ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store, s_stream->ssl, s_stream->up_parts.client_hello->sni, &(s_stream->up_parts.verify_param), error_str, sizeof(error_str), &(s_stream->up_parts.verify_result)); s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct; @@ -1333,7 +1333,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, else { //Do not perform cert verification on reused session. - s_upstream->is_server_cert_verify_passed=1; + s_upstream->is_server_cert_verify_passed=1; } if(mgr->log_master_key) { @@ -1478,7 +1478,7 @@ void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_s ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p); ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); } - + static int ossl_session_ticket_key_callback(SSL *ssl_conn, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, @@ -1494,14 +1494,14 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn, const char * sni = s_stream->peer->up_parts.client_hello->sni; struct sess_ticket_box * stek_box = s_stream->mgr->down_stek_box; struct sess_ticket_key cur_stek; - - unsigned char buf[33]={0}; + + unsigned char buf[33]={0}; ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY])); if (enc == 1) { - + sess_ticket_box_get_key_for_enc(stek_box, sni, &cur_stek); - /* encrypt session stek */ + /* encrypt session stek */ if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1) { TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed"); @@ -1514,7 +1514,7 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn, ret=STEK_ERROR; goto leave; } - if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1) + if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1) { TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed"); ret=STEK_ERROR; @@ -1523,10 +1523,10 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn, memcpy(name, cur_stek.name, sizeof(cur_stek.name)); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW])); ret=STEK_FOUND_FRESH; - } - else + } + else { - /* decrypt session stek */ + /* decrypt session stek */ ret=sess_ticket_box_get_key_for_dec(stek_box, sni, name, &cur_stek); if(ret==STEK_FOUND_FRESH||ret==STEK_FOUND_STALED) @@ -1536,8 +1536,8 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn, TFE_LOG_ERROR(logger, "HMAC_Init_ex() failed"); ret= STEK_ERROR; goto leave; - } - if (EVP_DecryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1) + } + if (EVP_DecryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1) { TFE_LOG_ERROR(logger, "EVP_DecryptInit_ex() failed"); ret= STEK_ERROR; @@ -1552,7 +1552,7 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn, ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND])); } } - + leave: return (int)ret; } @@ -1639,7 +1639,7 @@ static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); } - if (mgr->no_ssl3) + if (mgr->no_ssl3) { SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); } @@ -1780,7 +1780,7 @@ void downstream_ossl_init(struct ssl_stream *s_stream) ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream); assert(ret == 1); - + if (mgr->ssl_mode_release_buffers == 1) { /* lower memory footprint for idle connections */ @@ -1794,7 +1794,7 @@ void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx) { X509_free(ctx->origin_crt); ctx->origin_crt=NULL; - + if (ctx->f_ask_keyring != NULL) { future_destroy(ctx->f_ask_keyring); @@ -1842,9 +1842,9 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, struct ssl_stream * s_stream = ctx->downstream; struct ssl_upstream_parts* s_upstream= &(ctx->peer->up_parts); - struct ssl_mgr* mgr=s_stream->mgr; + struct ssl_mgr* mgr=s_stream->mgr; char error_str[TFE_STRING_MAX]={0}; - uint64_t jiffies_ms=0; + uint64_t jiffies_ms=0; unsigned long sslerr=0; if (events & BEV_EVENT_ERROR) { @@ -1857,7 +1857,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, s_stream->error=SSL_STREAM_R_CLIENT_PROTOCOL_ERROR; } else if(events & BEV_EVENT_EOF) - { + { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); ssl_stream_process_zero_eof(s_stream, mgr); } @@ -1868,14 +1868,14 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, } else if(events & BEV_EVENT_CONNECTED) { - + clock_gettime(CLOCK_MONOTONIC, &(ctx->end)); jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000; if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS) { - TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s", - jiffies_ms, - s_stream->tcp_stream->str_stream_info, + TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s", + jiffies_ms, + s_stream->tcp_stream->str_stream_info, s_upstream->client_hello->sni); } s_stream->connect_latency_ms=jiffies_ms; @@ -1895,7 +1895,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, // ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &svc_status); promise_success(p, ctx); } - + if(s_stream->error) { ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error)); @@ -1917,7 +1917,7 @@ void ask_keyring_on_succ(void * result, void * user) struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id); kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing. - + clock_gettime(CLOCK_MONOTONIC, &(ctx->start)); ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, kyr, ctx->peer, ctx->tcp_stream); @@ -1925,7 +1925,7 @@ void ask_keyring_on_succ(void * result, void * user) ctx->bev_down = bufferevent_openssl_socket_new(evbase, ctx->fd_downstream, ctx->downstream->ssl, BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); - + bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p); bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only @@ -2121,7 +2121,7 @@ complete: * Guarantees that SSL and the corresponding SSL_CTX are freed and the * socket is closed, eventually, or in the case of fatal errors, immediately. */ -void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev) +void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev) { UNUSED struct ssl_shutdown_ctx * sslshutctx = NULL; evutil_socket_t fd=-1; @@ -2165,8 +2165,8 @@ void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_bas bufferevent_setcb(ubev, NULL, NULL, NULL, NULL); bufferevent_free(ubev); } + ssl_stream_free(s_stream); - evutil_closesocket(fd); } int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file) @@ -2301,4 +2301,4 @@ int ssl_stream_get_string_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT o return -1; } return 0; -} \ No newline at end of file +} diff --git a/platform/src/tcp_stream.cpp b/platform/src/tcp_stream.cpp index 89ec256..fa33f6a 100644 --- a/platform/src/tcp_stream.cpp +++ b/platform/src/tcp_stream.cpp @@ -401,7 +401,11 @@ evutil_socket_t __conn_private_release_fd(struct tfe_conn_private * conn) static void __conn_private_destory(struct tfe_conn_private * conn) { - bufferevent_disable(conn->bev, EV_READ | EV_WRITE); + if (conn->bev) + { + bufferevent_disable(conn->bev, EV_READ | EV_WRITE); + bufferevent_free(conn->bev); + } if(conn->ratelimit_bucket) { @@ -409,10 +413,9 @@ static void __conn_private_destory(struct tfe_conn_private * conn) conn->ratelimit_bucket = NULL; } - bufferevent_free(conn->bev); if (conn->fd > 0) { - evutil_closesocket(conn->fd); + assert(evutil_closesocket(conn->fd) >= 0); } free(conn); @@ -423,8 +426,7 @@ static void __conn_private_destory_with_ssl(struct event_base * ev_base, struct tfe_conn_private * conn, struct ssl_stream * ssl_stream) { if (ssl_stream == NULL) return __conn_private_destory(conn); - ssl_stream_free_and_close_fd(ssl_stream, ev_base, conn->bev); - conn->fd = -1; + ssl_stream_free(ssl_stream, ev_base, conn->bev); return __conn_private_destory(conn); } @@ -950,14 +952,15 @@ void ssl_downstream_create_on_success(future_result_t * result, void * user) { struct tfe_stream_private * _stream = (struct tfe_stream_private *) user; struct ssl_stream * downstream = ssl_downstream_create_result_release_stream(result); + struct bufferevent * bev = ssl_downstream_create_result_release_bev(result); + _stream->defer_fd_downstream = 0; _stream->conn_downstream = __conn_private_create_by_bev(_stream, bev); _stream->ssl_downstream = downstream; future_destroy(_stream->future_downstream_create); _stream->future_downstream_create = NULL; - _stream->defer_fd_downstream = 0; assert(_stream->conn_downstream != NULL && _stream->conn_upstream != NULL); @@ -988,19 +991,17 @@ void ssl_downstream_create_on_fail(enum e_future_error err, const char * what, v void ssl_upstream_create_on_success(future_result_t * result, void * user) { struct tfe_stream_private * _stream = (struct tfe_stream_private *) user; - evutil_socket_t fd=-1; - enum ssl_stream_action ssl_action=ssl_upstream_create_result_release_action(result); - if(SSL_ACTION_PASSTHROUGH==ssl_action) + evutil_socket_t fd = -1; + enum ssl_stream_action ssl_action = ssl_upstream_create_result_release_action(result); + if (SSL_ACTION_PASSTHROUGH == ssl_action) { - _stream->tcp_passthough=1; - fd=ssl_upstream_create_result_release_fd(result); - _stream->conn_upstream=__conn_private_create_by_fd(_stream, fd); - _stream->conn_downstream=__conn_private_create_by_fd(_stream, _stream->defer_fd_downstream); + _stream->tcp_passthough = 1; + _stream->conn_upstream = __conn_private_create_by_fd(_stream, _stream->defer_fd_upstream); + _stream->conn_downstream = __conn_private_create_by_fd(_stream, _stream->defer_fd_downstream); __conn_private_enable(_stream->conn_downstream); __conn_private_enable(_stream->conn_upstream); - } - else if(SSL_ACTION_SHUTDOWN==ssl_action) + else if (SSL_ACTION_SHUTDOWN == ssl_action) { tfe_stream_destory(_stream); } @@ -1010,7 +1011,8 @@ void ssl_upstream_create_on_success(future_result_t * result, void * user) struct bufferevent * bev = ssl_upstream_create_result_release_bev(result); assert(upstream != NULL && bev != NULL); - /* Create connection ctx by bev */ + /* Create connection ctx by bev, fd's ownership is transfer to bev */ + _stream->defer_fd_upstream = 0; _stream->conn_upstream = __conn_private_create_by_bev(_stream, bev); _stream->ssl_upstream = upstream; @@ -1024,10 +1026,9 @@ void ssl_upstream_create_on_success(future_result_t * result, void * user) ssl_async_downstream_create(_stream->future_downstream_create, _stream->ssl_mgr, _stream->ssl_upstream, _stream->defer_fd_downstream, &_stream->head); } + future_destroy(_stream->future_upstream_create); _stream->future_upstream_create = NULL; - _stream->defer_fd_upstream = 0; - } void ssl_upstream_create_on_fail(enum e_future_error err, const char * what, void * user) @@ -1126,21 +1127,23 @@ void tfe_stream_destory(struct tfe_stream_private * stream) if (__is_ssl(stream) && stream->ssl_upstream) { - ssl_stream_free_and_close_fd(stream->ssl_upstream, ev_base, stream->conn_upstream->bev); + ssl_stream_free(stream->ssl_upstream, ev_base, stream->conn_upstream->bev); } if (__is_ssl(stream) && stream->ssl_downstream) { - ssl_stream_free_and_close_fd(stream->ssl_downstream, ev_base, stream->conn_downstream->bev); + ssl_stream_free(stream->ssl_downstream, ev_base, stream->conn_downstream->bev); } if (stream->conn_upstream) { + assert(stream->defer_fd_upstream <= 0); __conn_private_destory(stream->conn_upstream); } if (stream->conn_downstream) { + assert(stream->defer_fd_downstream <= 0); __conn_private_destory(stream->conn_downstream); } @@ -1327,13 +1330,21 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst if (_stream->session_type == STREAM_PROTO_PLAIN) { _stream->conn_downstream = __conn_private_create_by_fd(_stream, fd_downstream); + if (_stream->conn_downstream != NULL) + { + _stream->defer_fd_downstream = 0; + } + else + { + goto __errout; + } + _stream->conn_upstream = __conn_private_create_by_fd(_stream, fd_upstream); - - /* Defer FD has been transfer to conn_downstream/conn_upstream */ - _stream->defer_fd_downstream = 0; - _stream->defer_fd_upstream = 0; - - if (unlikely(_stream->conn_downstream == NULL || _stream->conn_upstream == NULL)) + if (_stream->conn_upstream != NULL) + { + _stream->defer_fd_downstream = 0; + } + else { goto __errout; } @@ -1364,6 +1375,12 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst return 0; __errout: + /* The fds not been accept by this function, clear up and release at caller */ + _stream->defer_fd_downstream = 0; + _stream->defer_fd_upstream = 0; + _stream->log_fd_downstream = 0; + _stream->log_fd_upstream = 0; + return -1; } diff --git a/script/CMakeLists.txt b/script/CMakeLists.txt index 04273ea..b269bf2 100644 --- a/script/CMakeLists.txt +++ b/script/CMakeLists.txt @@ -1,5 +1,4 @@ -install(PROGRAMS user/r2_tfe DESTINATION ./ COMPONENT Program) -install(PROGRAMS user/r3_tfe DESTINATION ./ COMPONENT Program) install(FILES sysctl/80-tfe.conf DESTINATION /etc/sysctl.d/ COMPONENT Program) +install(FILES service/tfe.service DESTINATION /usr/lib/systemd/system/ COMPONENT Program) install(FILES service/tfe-env.service DESTINATION /usr/lib/systemd/system/ COMPONENT Program) install(FILES service/tfe-env-config DESTINATION /etc/sysconfig/ COMPONENT Profile) diff --git a/script/service/tfe-env.service b/script/service/tfe-env.service index 6dc3f9c..6707505 100644 --- a/script/service/tfe-env.service +++ b/script/service/tfe-env.service @@ -1,5 +1,7 @@ [Unit] Description=Tango Frontend Engine - Running Environment Setup +Before=tfe.service +Wants=network-online.target [Service] EnvironmentFile=/etc/sysconfig/tfe-env-config @@ -10,7 +12,7 @@ RemainAfterExit=yes ExecStart=/bin/true ExecStop=/bin/true -# dataincoming interface +ExecStartPost=/usr/sbin/modprobe tfe-kmod ExecStartPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} address ${TFE_LOCAL_MAC_DATA_INCOMING} ExecStartPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} up ExecStartPost=/usr/sbin/ip addr flush dev ${TFE_DEVICE_DATA_INCOMING} @@ -41,6 +43,8 @@ ExecStopPost=/usr/sbin/ip -6 route del default via fd00::01 ExecStopPost=/usr/sbin/ip -6 route del local default dev lo table 102 ExecStopPost=/usr/sbin/ip addr del fd00::02/64 dev ${TFE_DEVICE_DATA_INCOMING} ExecStopPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} down +ExecStopPost=/usr/sbin/modprobe -r tfe-kmod [Install] +RequiredBy=tfe.service WantedBy=multi-user.target diff --git a/script/service/tfe.service b/script/service/tfe.service new file mode 100644 index 0000000..b5b39a0 --- /dev/null +++ b/script/service/tfe.service @@ -0,0 +1,20 @@ +[Unit] +Description=Tango Frontend Engine +Requires=tfe-env.service +After=tfe-env.service + +[Service] +Type=simple +ExecStart=/opt/tsg/tfe/bin/tfe +TimeoutSec=180s +RestartSec=10s +Restart=always +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target