From bc41051da294276f8f27a8cf8edbdb6968c8393e Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sat, 1 Jun 2019 20:28:07 +0800 Subject: [PATCH] =?UTF-8?q?ssl=20stream=E5=92=8Cssl=20policy=E5=AF=B9?= =?UTF-8?q?=E6=8E=A5tfe=5Fcmsg=5Fxx=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/include/ssl_stream.h | 3 +- common/include/tfe_cmsg.h | 11 +- common/include/tfe_stream.h | 1 + common/src/tfe_cmsg.cpp | 6 +- platform/include/internal/ssl_stream_core.h | 18 +- platform/src/proxy.cpp | 2 +- platform/src/ssl_stream.cpp | 162 ++++++++++++------ platform/src/tcp_stream.cpp | 5 +- plugin/business/ssl-policy/src/ssl_policy.cpp | 8 +- 9 files changed, 147 insertions(+), 69 deletions(-) diff --git a/common/include/ssl_stream.h b/common/include/ssl_stream.h index cbe7f6a..7fad2fe 100644 --- a/common/include/ssl_stream.h +++ b/common/include/ssl_stream.h @@ -4,7 +4,7 @@ struct ssl_stream; enum ssl_stream_action { - SSL_ACTION_PASSTHROUGH, + SSL_ACTION_PASSTHROUGH=0, SSL_ACTION_INTERCEPT, SSL_ACTION_SHUTDOWN }; @@ -12,6 +12,7 @@ typedef enum ssl_stream_action ssl_stream_new_hook(struct ssl_stream *upstream, enum SSL_STREAM_OPT { + SSL_STREAM_OPT_INTERCEPT_POLICY_ID, SSL_STREAM_OPT_IS_EV_CERT, //0:FALSE, 1:TRUE. SSL_STREAM_OPT_IS_CT_CERT, //0:FALSE, 1:TRUE. SSL_STREAM_OPT_IS_MUTUAL_AUTH, //0:FALSE, 1:TRUE. diff --git a/common/include/tfe_cmsg.h b/common/include/tfe_cmsg.h index 31d7246..fa2ca97 100644 --- a/common/include/tfe_cmsg.h +++ b/common/include/tfe_cmsg.h @@ -29,12 +29,21 @@ enum tfe_cmsg_tlv_type TFE_CMSG_POLICY_ID = 0x10, TFE_CMSG_STREAM_TRACE_ID = 0x11, + + TFE_CMSG_SSL_INTERCEPT_STATE, //size uint64_t, 0-passthrough, 1-intercept, 2-shutdown, referer from enum ssl_stream_action + TFE_CMSG_SSL_UPSTREAM_LATENCY, //size uint64_t, milisecond + TFE_CMSG_SSL_DOWNSTREAM_LATENCY, //size uint64_t, milisecond + TFE_CMSG_SSL_UPSTREAM_VERSION, //string, SSLv3 TLSv1 TLSv1.1 TLSv1.2 TLSv1.3 unknown + TFE_CMSG_SSL_DOWNSTREAM_VERSION, + TFE_CMSG_SSL_PINNING_STATE, //size uint64_t, 0-not pinning 1-pinning 2-maybe pinning + TFE_CMSG_SSL_CERT_VERIFY, + TFE_CMSG_SSL_ERROR //string }; struct tfe_cmsg* tfe_cmsg_init(); void tfe_cmsg_destroy(struct tfe_cmsg *cmsg); -int tfe_cmsg_get_value(struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, char * out_value, +int tfe_cmsg_get_value(struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, unsigned char * out_value, size_t sz_out_value_buf, uint16_t * out_size); int tfe_cmsg_set(struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, const unsigned char * value, uint16_t size); uint16_t tfe_cmsg_serialize_size_get(struct tfe_cmsg *cmsg); diff --git a/common/include/tfe_stream.h b/common/include/tfe_stream.h index 97329ad..50c839d 100644 --- a/common/include/tfe_stream.h +++ b/common/include/tfe_stream.h @@ -41,6 +41,7 @@ struct tfe_stream const char * str_stream_info; struct tfe_stream_addr * addr; enum tfe_stream_proto proto; + unsigned int thread_id; struct tfe_conn upstream; struct tfe_conn downstream; }; diff --git a/common/src/tfe_cmsg.cpp b/common/src/tfe_cmsg.cpp index 3a3881a..3823c32 100644 --- a/common/src/tfe_cmsg.cpp +++ b/common/src/tfe_cmsg.cpp @@ -75,12 +75,12 @@ int tfe_cmsg_set(struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, const unsi return 0; } -int tfe_cmsg_get_value(struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, char * out_value, +int tfe_cmsg_get_value(struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, unsigned char * out_value, size_t sz_out_value_buf, uint16_t * out_size) { struct tfe_cmsg_tlv *tlv; int result = 0; - int value_length = 0; + size_t value_length = 0; if (unlikely(type >= TFE_CMSG_TLV_NR_MAX)) { @@ -96,7 +96,7 @@ int tfe_cmsg_get_value(struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, char } value_length = tlv->length - sizeof(struct tfe_cmsg_tlv); - if (unlikely(sz_out_value_buf < value_length)) + if (unlikely(sz_out_value_buf < value_length)) { result = -ENOBUFS; goto errout; diff --git a/platform/include/internal/ssl_stream_core.h b/platform/include/internal/ssl_stream_core.h index e922a6e..c72c145 100644 --- a/platform/include/internal/ssl_stream_core.h +++ b/platform/include/internal/ssl_stream_core.h @@ -4,6 +4,18 @@ #include #include +enum ssl_stream_error +{ + SSL_STREAM_R_NO_ERROR=0, + SSL_STREAM_R_SERVER_CLOSED, + SSL_STREAM_R_CLIENT_CLOSED, + SSL_STREAM_R_CONNECT_SERVER_TIMEOUT, + SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT, + SSL_STREAM_R_SERVER_PROTOCOL_ERROR, + SSL_STREAM_R_CLIENT_PROTOCOL_ERROR, + __SSL_STREAM_R_MAX +}; + struct ssl_mgr; @@ -20,14 +32,16 @@ evutil_socket_t ssl_upstream_create_result_release_fd(future_result_t * result); struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result); struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result); void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, - evutil_socket_t fd_downstream, unsigned int thread_id); + evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream); + struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result); struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result); void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, - evutil_socket_t fd_downstream, unsigned int thread_id); + evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream); + void ssl_stream_free_and_close_fd(struct ssl_stream * stream, struct event_base * evbase, evutil_socket_t fd); const char* ssl_stream_dump_info(struct ssl_stream *stream, char* buffer, size_t sz); diff --git a/platform/src/proxy.cpp b/platform/src/proxy.cpp index 69c7ac4..4c594d3 100644 --- a/platform/src/proxy.cpp +++ b/platform/src/proxy.cpp @@ -108,7 +108,7 @@ int tfe_proxy_fds_accept(struct tfe_proxy * ctx, int fd_downstream, int fd_upstr enum tfe_stream_proto stream_protocol; uint16_t __size; - int result = tfe_cmsg_get_value(cmsg, TFE_CMSG_TCP_RESTORE_PROTOCOL, (char *)&stream_protocol, + int result = tfe_cmsg_get_value(cmsg, TFE_CMSG_TCP_RESTORE_PROTOCOL, (unsigned char *)&stream_protocol, sizeof(stream_protocol), &__size); if (unlikely(result < 0)) diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index d5256c3..8c2e68f 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -185,7 +185,7 @@ struct ssl_upstream_parts int apln_enabled; int keyring_id; struct ssl_chello * client_hello; - int is_server_cert_verify_passed; + int is_server_cert_verify_passed; }; struct ssl_downstream_parts { @@ -203,10 +203,14 @@ struct ssl_stream }; int ssl_min_version, ssl_max_version, negotiated_version; const unsigned char* alpn_selected; //reference to SSL_ALPN_HTTP_2/SSL_ALPN_HTTP_1_1 + struct tfe_stream* tcp_stream; + uint64_t connect_latency_ms; + struct ssl_stream* peer; socklen_t addrlen; struct sockaddr_storage addr; struct __ssl_stream_debug _do_not_use; + enum ssl_stream_error error; }; @@ -230,16 +234,17 @@ struct ssl_connect_server_ctx evutil_socket_t fd_upstream; evutil_socket_t fd_downstream; - unsigned int thread_id; + + struct tfe_stream* tcp_stream; + struct future * f_peek_chello; struct timespec start,end; }; struct ssl_connect_client_ctx { - unsigned int thread_id; - int keyring_id; - struct ssl_stream * origin_ssl; + struct tfe_stream* tcp_stream; + struct ssl_stream * peer; X509 * origin_crt; int is_origin_crt_verify_passed; struct ssl_mgr * ssl_mgr; @@ -307,7 +312,18 @@ int sslver_str2num(const char * version_str) return sslversion; } - +const char* ssl_stream_get_error_string(enum ssl_stream_error error) +{ + const char* ssl_err_str[__SSL_STREAM_R_MAX]; + ssl_err_str[SSL_STREAM_R_NO_ERROR]="OK"; + ssl_err_str[SSL_STREAM_R_SERVER_CLOSED]="server-side closed"; + ssl_err_str[SSL_STREAM_R_CLIENT_CLOSED]="client-side closed"; + ssl_err_str[SSL_STREAM_R_CONNECT_SERVER_TIMEOUT]="server-side timeout"; + ssl_err_str[SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT]="client-side timeout"; + ssl_err_str[SSL_STREAM_R_SERVER_PROTOCOL_ERROR]="server-side protocol errors"; + ssl_err_str[SSL_STREAM_R_CLIENT_PROTOCOL_ERROR]="client-side protocol errors"; + return ssl_err_str[error]; +} /* * Garbage collection handler. */ @@ -463,7 +479,7 @@ struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result) } struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir, - struct ssl_chello * client_hello, struct keyring * kyr, struct ssl_stream * peer) + struct ssl_chello * client_hello, struct keyring * kyr, struct ssl_stream * peer, struct tfe_stream* tcp_stream) { UNUSED int ret = 0; @@ -475,7 +491,7 @@ struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enu s_stream->ssl_max_version=mgr->ssl_max_version; s_stream->ssl_min_version=mgr->ssl_min_version; s_stream->peer=peer; - + s_stream->tcp_stream=tcp_stream; ret = getpeername(fd, (struct sockaddr *) (&s_stream->addr), &(s_stream->addrlen)); switch (dir) { @@ -1098,6 +1114,21 @@ unsigned long ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir d return ret_sslerr; } +static void ssl_stream_set_cmsg_string(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, const char* value_str) +{ + struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream); + int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)value_str, (uint16_t)strlen(value_str)); + assert(ret==0); + return; +} +static void ssl_stream_set_cmsg_integer(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, uint64_t value_int) +{ + struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream); + int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)&value_int, (uint16_t)sizeof(value_int)); + assert(ret==0); + return; +} + /* * Callback for meta events on the up- and downstream connection bufferevents. * Called when EOF has been reached, a connection has been made, and on errors. @@ -1114,8 +1145,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, SSL_SESSION * ssl_sess = NULL; char error_str[TFE_STRING_MAX]; const char* sni=s_upstream->client_hello->sni?s_upstream->client_hello->sni:"null"; - long jiffies_ms; - char* addr_string=NULL; + uint64_t jiffies_ms; unsigned long sslerr=0; if (events & BEV_EVENT_ERROR) { @@ -1126,20 +1156,17 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, s_upstream->svc_status.has_protocol_errors=1; ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, &(s_stream->up_parts.svc_status)); } - snprintf(error_str, sizeof(error_str), "connect to original server failed : sni=%s", sni); - promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str); + s_stream->error=SSL_STREAM_R_SERVER_PROTOCOL_ERROR; } else if(events & BEV_EVENT_EOF) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); - snprintf(error_str, sizeof(error_str), "original server closed : sni=%s", sni); - promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str); + s_stream->error=SSL_STREAM_R_SERVER_CLOSED; } else if(events & BEV_EVENT_TIMEOUT) { - ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); - snprintf(error_str, sizeof(error_str), "timeout : sni=%s", sni); - promise_failed(p, FUTURE_ERROR_TIMEOUT, error_str); + ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); + s_stream->error=SSL_STREAM_R_CONNECT_SERVER_TIMEOUT; } else if(events & BEV_EVENT_CONNECTED) { @@ -1149,11 +1176,10 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000; if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS) { - addr_string=tfe_string_addr_create_by_fd(ctx->fd_upstream, CONN_DIR_UPSTREAM); - TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s", jiffies_ms, addr_string, sni); - free(addr_string); - addr_string=NULL; + TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s", jiffies_ms, s_stream->tcp_stream->str_stream_info, sni); } + s_stream->connect_latency_ms=jiffies_ms; + ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_UPSTREAM_LATENCY, jiffies_ms); if(!SSL_session_reused(s_stream->ssl)) { @@ -1169,7 +1195,8 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct; s_upstream->svc_status.is_ev=s_upstream->verify_result.is_ev; ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, &(s_upstream->svc_status)); - TFE_LOG_DEBUG(mgr->logger, "SNI: %s hostmatch:%d, ct:%d, ev:%d", + TFE_LOG_DEBUG(mgr->logger, "stream:%s sni:%s hostmatch:%d, ct:%d, ev:%d", + s_stream->tcp_stream->str_stream_info, s_upstream->client_hello->sni, s_upstream->verify_result.is_hostmatched, s_upstream->verify_result.is_ct, @@ -1226,8 +1253,16 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, } } s_stream->negotiated_version=SSL_version(s_stream->ssl); + ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_UPSTREAM_VERSION, SSL_get_version(s_stream->ssl)); + ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_CERT_VERIFY, s_upstream->is_server_cert_verify_passed); promise_success(p, ctx); } + if(s_stream->error) + { + ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error)); + snprintf(error_str, sizeof(error_str), "%s, sni=%s", ssl_stream_get_error_string(s_stream->error), sni); + promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str); + } wrap_ssl_connect_server_ctx_free(ctx); return; } @@ -1236,7 +1271,7 @@ static void peek_chello_on_succ(future_result_t * result, void * user) { struct promise * p = (struct promise *) user; struct ssl_connect_server_ctx* ctx = (struct ssl_connect_server_ctx *) promise_get_ctx(p); - struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->thread_id); + struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id); struct ssl_stream* s_stream=NULL; struct ssl_chello* chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. if(chello->sni==NULL) @@ -1246,7 +1281,7 @@ static void peek_chello_on_succ(future_result_t * result, void * user) int ret=0; struct ssl_service_status* svc_status=NULL; clock_gettime(CLOCK_MONOTONIC, &(ctx->start)); - s_stream= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL, NULL); + s_stream= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL, NULL, ctx->tcp_stream); svc_status=&s_stream->up_parts.svc_status; ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, svc_status); if(ret==1) @@ -1259,6 +1294,7 @@ static void peek_chello_on_succ(future_result_t * result, void * user) svc_status->is_ct, svc_status->is_ev); } + ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, svc_status->pinning_status); if(ctx->mgr->on_new_upstream_cb) { s_stream->up_parts.action=ctx->mgr->on_new_upstream_cb(s_stream, ctx->mgr->upstream_cb_param); @@ -1267,6 +1303,7 @@ static void peek_chello_on_succ(future_result_t * result, void * user) { s_stream->up_parts.action=SSL_ACTION_INTERCEPT; } + ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_INTERCEPT_STATE, s_stream->up_parts.action); ctx->s_stream = s_stream; if(s_stream->up_parts.action==SSL_ACTION_PASSTHROUGH) { @@ -1301,7 +1338,7 @@ static void peek_chello_on_fail(enum e_future_error err, const char * what, void } void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, - evutil_socket_t fd_downstream, unsigned int thread_id) + evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream) { struct promise * p = future_to_promise(f); struct ssl_connect_server_ctx * ctx = ALLOC(struct ssl_connect_server_ctx, 1); @@ -1315,10 +1352,10 @@ void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_s promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream fd closed"); return; } - struct event_base* evbase=tfe_proxy_get_work_thread_evbase(thread_id); + struct event_base* evbase=tfe_proxy_get_work_thread_evbase(tcp_stream->thread_id); ctx->fd_downstream = fd_downstream; ctx->fd_upstream = fd_upstream; - ctx->thread_id = thread_id; + ctx->tcp_stream = tcp_stream; ctx->mgr = mgr; promise_set_ctx(p, ctx, wrap_ssl_connect_server_ctx_free); @@ -1689,12 +1726,11 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p); struct ssl_stream * s_stream = ctx->downstream; - struct ssl_upstream_parts* s_upstream= &(ctx->origin_ssl->up_parts); + struct ssl_upstream_parts* s_upstream= &(ctx->peer->up_parts); struct ssl_mgr* mgr=s_stream->mgr; - char* addr_string=NULL; const char* sni=s_upstream->client_hello->sni?s_upstream->client_hello->sni:"null"; char error_str[TFE_STRING_MAX]={0}; - long jiffies_ms=0; + uint64_t jiffies_ms=0; unsigned long sslerr=0; if (events & BEV_EVENT_ERROR) { @@ -1705,6 +1741,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, s_upstream->verify_result.is_hostmatched) { s_upstream->svc_status.pinning_status=PINNING_ST_PINNING; + ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING); ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, &s_upstream->svc_status); } else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN) @@ -1712,8 +1749,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, s_upstream->svc_status.has_protocol_errors=1; ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, &s_upstream->svc_status); } - snprintf(error_str, sizeof(error_str), "connect to client failed : sni=%s", sni); - promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str); + s_stream->error=SSL_STREAM_R_CLIENT_PROTOCOL_ERROR; } else if(events & BEV_EVENT_EOF) { @@ -1721,16 +1757,15 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, if(s_upstream->verify_result.is_hostmatched && s_upstream->is_server_cert_verify_passed ) { s_upstream->svc_status.pinning_status=PINNING_ST_MAYBE_PINNING; + ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_MAYBE_PINNING); ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, &(s_stream->peer->up_parts.svc_status)); } - snprintf(error_str, sizeof(error_str), "client side closed : sni=%s", sni); - promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str); + s_stream->error=SSL_STREAM_R_CLIENT_CLOSED; } else if(events & BEV_EVENT_TIMEOUT) { ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); - snprintf(error_str, sizeof(error_str), "timeout : sni=%s", sni); - promise_failed(p, FUTURE_ERROR_TIMEOUT, error_str); + s_stream->error=SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT; } else if(events & BEV_EVENT_CONNECTED) { @@ -1739,21 +1774,29 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000; if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS) { - addr_string=tfe_string_addr_create_by_fd(ctx->fd_downstream, CONN_DIR_DOWNSTREAM); - TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s", jiffies_ms, addr_string, sni); - free(addr_string); - addr_string=NULL; + TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s", jiffies_ms, s_stream->tcp_stream->str_stream_info, sni); } + s_stream->connect_latency_ms=jiffies_ms; + ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_DOWNSTREAM_LATENCY, jiffies_ms); bufferevent_disable(ctx->bev_down, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev_down, NULL, NULL, NULL, NULL); //leave a clean bev for on_success if(mgr->log_master_key) { - log_ssl_master_key(ctx->downstream->ssl, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key); + log_ssl_master_key(s_stream->ssl, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key); } - ctx->downstream->negotiated_version=SSL_version(ctx->downstream->ssl); + s_stream->negotiated_version=SSL_version(s_stream->ssl); + ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_DOWNSTREAM_VERSION, SSL_get_version(s_stream->ssl)); promise_success(p, ctx); } + + if(s_stream->error) + { + ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error)); + snprintf(error_str, sizeof(error_str), "%s : sni=%s", ssl_stream_get_error_string(s_stream->error), sni); + promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str); + } + ssl_connect_client_ctx_free(ctx); return; } @@ -1765,13 +1808,13 @@ void ask_keyring_on_succ(void * result, void * user) struct keyring * kyr = NULL; struct ssl_mgr * mgr = ctx->ssl_mgr; - struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->thread_id); + struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id); kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing. clock_gettime(CLOCK_MONOTONIC, &(ctx->start)); ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, - kyr, ctx->origin_ssl); + kyr, ctx->peer, ctx->tcp_stream); downstream_ossl_init(ctx->downstream); ctx->bev_down = bufferevent_openssl_socket_new(evbase, ctx->fd_downstream, ctx->downstream->ssl, BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE); @@ -1798,22 +1841,21 @@ void ask_keyring_on_fail(enum e_future_error error, const char * what, void * us * Create a SSL stream for the incoming connection, based on the upstream. */ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream, - evutil_socket_t fd_downstream, unsigned int thread_id) + evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream) { assert(upstream->dir == CONN_DIR_UPSTREAM); const char* sni=NULL; - struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1); - ctx->keyring_id = upstream->up_parts.keyring_id; + struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1); ctx->ssl_mgr = mgr; ctx->fd_downstream = fd_downstream; - ctx->thread_id = thread_id; - struct event_base * evbase=tfe_proxy_get_work_thread_evbase(thread_id); - struct evdns_base* dnsbase=tfe_proxy_get_work_thread_dnsbase(thread_id); + ctx->tcp_stream = tcp_stream; + struct event_base * evbase=tfe_proxy_get_work_thread_evbase(tcp_stream->thread_id); + struct evdns_base* dnsbase=tfe_proxy_get_work_thread_dnsbase(tcp_stream->thread_id); if (upstream != NULL) { - ctx->origin_ssl = upstream; + ctx->peer = upstream; ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl); sni=upstream->up_parts.client_hello->sni; } @@ -1823,7 +1865,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ctx->f_ask_keyring = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p); ctx->is_origin_crt_verify_passed = upstream->up_parts.is_server_cert_verify_passed; - key_keeper_async_ask(ctx->f_ask_keyring, mgr->key_keeper, sni, ctx->keyring_id, ctx->origin_crt, ctx->is_origin_crt_verify_passed, + key_keeper_async_ask(ctx->f_ask_keyring, mgr->key_keeper, sni, upstream->up_parts.keyring_id, ctx->origin_crt, ctx->is_origin_crt_verify_passed, evbase, dnsbase); return; } @@ -2035,9 +2077,12 @@ int ssl_stream_set_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT } return 1; } -int ssl_stream_get_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, int *opt_val) +int ssl_stream_get_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, int *opt_val) { struct ssl_service_status* svc=&upstream->up_parts.svc_status; + struct tfe_cmsg *cmsg=NULL; + int ret=0; + uint16_t out_size=0; switch(opt_type) { case SSL_STREAM_OPT_IS_EV_CERT: @@ -2055,11 +2100,16 @@ int ssl_stream_get_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT case SSL_STREAM_OPT_HAS_PROTOCOL_ERRORS: *opt_val=svc->has_protocol_errors; break; - + case SSL_STREAM_OPT_INTERCEPT_POLICY_ID: + cmsg=tfe_stream_get0_cmsg(upstream->tcp_stream); + ret=tfe_cmsg_get_value(cmsg, TFE_CMSG_POLICY_ID, (unsigned char*)opt_val, sizeof(*opt_val), &out_size); + assert(ret==0); + assert(out_size==sizeof(*opt_val)); + break; default: - return 0; + return -1; } - return 1; + return 0; } diff --git a/platform/src/tcp_stream.cpp b/platform/src/tcp_stream.cpp index 59eaa38..9b23a3c 100644 --- a/platform/src/tcp_stream.cpp +++ b/platform/src/tcp_stream.cpp @@ -942,7 +942,7 @@ void ssl_upstream_create_on_success(future_result_t * result, void * user) ssl_downstream_create_on_fail, _stream); ssl_async_downstream_create(_stream->future_downstream_create, _stream->ssl_mgr, - _stream->ssl_upstream, _stream->defer_fd_downstream, _stream->thread_ref->thread_id); + _stream->ssl_upstream, _stream->defer_fd_downstream, &_stream->head); } future_destroy(_stream->future_upstream_create); _stream->future_upstream_create = NULL; @@ -967,6 +967,7 @@ struct tfe_stream * tfe_stream_create(struct tfe_proxy * pxy, struct tfe_thread_ struct tfe_stream_private * _stream = ALLOC(struct tfe_stream_private, 1); TFE_PROXY_STAT_INCREASE(STAT_STREAM_OPEN, 1); + _stream->head.thread_id=thread_ctx->thread_id; _stream->thread_ref = thread_ctx; _stream->proxy_ref = pxy; _stream->stream_logger = pxy->logger; @@ -1266,7 +1267,7 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst /* Defer setup conn_downstream & conn_upstream in async callbacks. */ ssl_async_upstream_create(_stream->future_upstream_create, - _stream->ssl_mgr, fd_upstream, fd_downstream, _stream->thread_ref->thread_id); + _stream->ssl_mgr, fd_upstream, fd_downstream, &_stream->head); TFE_PROXY_STAT_INCREASE(STAT_STREAM_TCP_SSL, 1); } diff --git a/plugin/business/ssl-policy/src/ssl_policy.cpp b/plugin/business/ssl-policy/src/ssl_policy.cpp index 34d9089..6921f31 100644 --- a/plugin/business/ssl-policy/src/ssl_policy.cpp +++ b/plugin/business/ssl-policy/src/ssl_policy.cpp @@ -178,6 +178,8 @@ enum ssl_stream_action ssl_policy_enforce(struct ssl_stream *upstream, void* u_p UNUSED int ret=0; int policy_id=0; char policy_id_str[16]={0}; + ret=ssl_stream_get_integer_opt(upstream, SSL_STREAM_OPT_INTERCEPT_POLICY_ID, &policy_id); + assert(ret==0); snprintf(policy_id_str, sizeof(policy_id_str), "%d", policy_id); param=(struct intercept_param *)Maat_plugin_get_EX_data(enforcer->maat, enforcer->table_id, policy_id_str); if(param==NULL) @@ -206,13 +208,13 @@ enum ssl_stream_action ssl_policy_enforce(struct ssl_stream *upstream, void* u_p ret=ssl_stream_set_integer_opt(upstream, SSL_STREAM_OPT_KEYRING_ID, param->keyring); ret=ssl_stream_get_integer_opt(upstream, SSL_STREAM_OPT_PINNING_STATUS, &pinning_staus); - assert(ret==1); + assert(ret==0); ret=ssl_stream_get_integer_opt(upstream, SSL_STREAM_OPT_IS_EV_CERT, &is_ev); - assert(ret==1); + assert(ret==0); ret=ssl_stream_get_integer_opt(upstream, SSL_STREAM_OPT_IS_MUTUAL_AUTH, &is_mauth); ret=ssl_stream_get_integer_opt(upstream, SSL_STREAM_OPT_IS_CT_CERT, &is_ct); ret=ssl_stream_get_integer_opt(upstream, SSL_STREAM_OPT_HAS_PROTOCOL_ERRORS, &has_error); - assert(ret=1); + assert(ret=0); if( (pinning_staus==1 && param->bypass_pinning) || (is_mauth && param->bypass_mutual_auth) || (is_ev && param->bypass_ev_cert) ||