From baa409ecc8d557066f871be47d1a0d08763c6aa2 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Fri, 31 Aug 2018 19:59:22 +0800 Subject: [PATCH] =?UTF-8?q?ssl=20stream=E5=A2=9E=E5=8A=A0=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/include/tfe_utils.h | 2 + platform/CMakeLists.txt | 13 +++- platform/src/ssl_sess_cache.cpp | 15 ++-- platform/src/ssl_stream.cpp | 126 ++++++++++++++++++++++++++++---- vendor/CMakeLists.txt | 4 + 5 files changed, 137 insertions(+), 23 deletions(-) diff --git a/common/include/tfe_utils.h b/common/include/tfe_utils.h index 42354c3..7fd4801 100644 --- a/common/include/tfe_utils.h +++ b/common/include/tfe_utils.h @@ -71,6 +71,8 @@ do { MESA_handle_runtime_log(handler, RLOG_LV_DEBUG, "tfe", fmt, ##__VA_ARGS__); }) #endif +#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED) +#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED) int addr_sock_to_layer(struct sockaddr * sock_addr, int sockaddrlen, struct layer_addr * layer_addr); int addr_layer_to_sock(struct layer_addr * layer_addr, struct sockaddr * sock_addr); char* tfe_strdup(const char* s); diff --git a/platform/CMakeLists.txt b/platform/CMakeLists.txt index c0eb6e0..07e8f43 100644 --- a/platform/CMakeLists.txt +++ b/platform/CMakeLists.txt @@ -5,7 +5,16 @@ target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal) target_link_libraries(tfe common) -target_link_libraries(tfe pthread dl openssl-ssl-static openssl-crypto-static pthread libevent-static - libevent-static-openssl libevent-static-pthreads MESA_handle_logger MESA_prof_load MESA_htable wiredcfg) +target_link_libraries(tfe pthread dl + openssl-ssl-static + openssl-crypto-static + pthread libevent-static + libevent-static-openssl + libevent-static-pthreads + MESA_handle_logger + MESA_prof_load + MESA_htable wiredcfg + MESA_field_stat + plugin_deliver) install(TARGETS tfe RUNTIME DESTINATION ./) diff --git a/platform/src/ssl_sess_cache.cpp b/platform/src/ssl_sess_cache.cpp index 9f6687f..9af4130 100644 --- a/platform/src/ssl_sess_cache.cpp +++ b/platform/src/ssl_sess_cache.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -189,9 +190,7 @@ void up_session_set(struct sess_cache * cache, struct sockaddr * addr, socklen_t struct asn1_sess * asn1 = NULL; asn1 = ssl_sess_serialize(sess); - struct sess_set_args set_args; - set_args.hash = cache->hash; - set_args.new_sess = asn1; + struct sess_set_args set_args{.hash = cache->hash, .new_sess = asn1}; no_use = MESA_htable_search_cb(cache->hash, key, key_size, sess_cache_set_cb, &set_args, &cb_ret); if (cb_ret == SESS_CACHE_UPDATE_OLD) { @@ -217,12 +216,12 @@ SSL_SESSION * up_session_get(struct sess_cache * cache, struct sockaddr * addr, key = NULL; if (cb_ret == 1) { - cache->hit_cnt++; + ATOMIC_INC(&(cache->hit_cnt)); return sess; } else { - cache->miss_cnt++; + ATOMIC_INC(&(cache->miss_cnt)); return NULL; } } @@ -263,12 +262,12 @@ SSL_SESSION * down_session_get(struct sess_cache * cache, const unsigned char * no_use = MESA_htable_search_cb(cache->hash, id, (unsigned int) idlen, sess_cache_get_cb, &sess, &cb_ret); if (cb_ret == 1) { - cache->hit_cnt++; + ATOMIC_INC(&(cache->hit_cnt)); return sess; } else { - cache->miss_cnt++; + ATOMIC_INC(&(cache->miss_cnt)); return NULL; } } @@ -281,7 +280,7 @@ void down_session_del(struct sess_cache * cache, const SSL_SESSION * sess) int ret = MESA_htable_del(cache->hash, id, len, NULL); if (ret != MESA_HTABLE_RET_OK) { - cache->del_err++; + ATOMIC_INC(&(cache->del_err)); } return; } diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index c24a063..0ad6fdc 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -47,7 +47,19 @@ */ #define DFLT_CURVE "prime256v1" - +enum ssl_stream_stat +{ + SSL_UP_NUM, + SSL_UP_ERR, + SSL_UP_CLOSED, + SSL_DOWN_NUM, + SSL_DOWN_ERR, + SSL_DOWN_CLOSED, + SSL_NO_CHELLO, + SSL_NO_SNI, + SSL_FAKE_CRT, + SSL_STAT_MAX +}; struct ssl_mgr { unsigned int sslcomp; @@ -61,7 +73,7 @@ struct ssl_mgr int sslversion; char ssl_session_context[8]; - unsigned int cache_slot_num; + unsigned int cache_slots; unsigned int sess_expire_seconds; struct sess_cache * down_sess_cache; @@ -70,15 +82,20 @@ struct ssl_mgr char default_ciphers[TFE_STRING_MAX]; DH * dh; char * ecdhcurve; - char * crlurl; + char * crl_url; uint8_t ssl_mode_release_buffers; - void * logger; char trust_CA_file[TFE_STRING_MAX]; char trust_CA_dir[TFE_STRING_MAX]; X509_STORE * trust_CA_store; struct key_keeper * keeper_of_keys; struct event_base * ev_base_gc; + struct event * gcev; + + void * logger; + screen_stat_handle_t* fs_handle; + long long stat_val[SSL_STAT_MAX]; + int fs_id[SSL_STAT_MAX]; }; struct __ssl_stream_debug @@ -156,7 +173,30 @@ struct ssl_shutdown_ctx struct event * ev; unsigned int retries; }; - +struct fs_spec +{ + enum ssl_stream_stat id; + const char* name; +}; +void ssl_stat_init(struct ssl_mgr * mgr) +{ + int i=0; + const char* spec[SSL_STAT_MAX]= { + [SSL_UP_NUM]="ssl_up", + [SSL_UP_ERR]="ssl_up_err", + [SSL_UP_CLOSED]="ssl_up_close", + [SSL_DOWN_NUM]="ssl_down", + [SSL_DOWN_ERR]="ssl_down_err", + [SSL_DOWN_CLOSED]="ssl_down_close", + [SSL_NO_CHELLO]="ssl_no_chello", + [SSL_NO_SNI]="ssl_no_sni", + [SSL_FAKE_CRT]="ssl_fake_crt"}; + for(i=0;ifs_id[i]=FS_register(mgr->fs_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,spec[i]); + } + return; +} static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt); static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * chello, evutil_socket_t fd); static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr); @@ -290,16 +330,46 @@ void ssl_manager_destroy(struct ssl_mgr * mgr) X509_STORE_free(mgr->trust_CA_store); mgr->trust_CA_store = NULL; } + if(mgr->down_sess_cache!=NULL) + { + ssl_sess_cache_destroy(mgr->down_sess_cache); + } + if(mgr->up_sess_cache!=NULL) + { + ssl_sess_cache_destroy(mgr->up_sess_cache); + } + if(mgr->gcev!=NULL) + { + event_free(mgr->gcev); + } free(mgr); } +/* + * Garbage collection handler. + */ +static void +ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) +{ + struct ssl_mgr *mgr=(struct ssl_mgr *)arg; + int i=0; + return; + for(i=0;ifs_handle, i, 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i]))); + } + return; +} struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, struct event_base * ev_base_gc, void * logger, screen_stat_handle_t * fs) { + struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here. struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1); int ret = 0, value = 0; char version_str[TFE_SYMBOL_MAX]; mgr->logger = logger; + mgr->ev_base_gc=ev_base_gc; + mgr->fs_handle=fs; MESA_load_profile_string_def(ini_profile, section, "ssl_version", version_str, sizeof(version_str), "tls12"); mgr->sslversion = sslver_str2num(version_str); @@ -322,11 +392,11 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section - MESA_load_profile_uint_def(ini_profile, section, "session_cache_slot_num", &(mgr->cache_slot_num), 4 * 1024 * 1024); - MESA_load_profile_uint_def(ini_profile, section, "session_cache_slot_num", &(mgr->sess_expire_seconds), 30 * 60); + MESA_load_profile_uint_def(ini_profile, section, "session_cache_slots", &(mgr->cache_slots), 4 * 1024 * 1024); + MESA_load_profile_uint_def(ini_profile, section, "session_cache_slots", &(mgr->sess_expire_seconds), 30 * 60); - mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); - mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slot_num, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); + mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); + mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); mgr->keeper_of_keys = key_keeper_init(ini_profile, section, logger); #if 0 if (mgr->keeper_of_keys == NULL) @@ -364,6 +434,16 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section } memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context)); + + //ssl_stat_init(mgr); + + mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr); + if (!mgr->gcev) + { + goto error_out; + } + evtimer_add(mgr->gcev, &gc_delay); + return mgr; error_out: @@ -694,11 +774,13 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v if (events & BEV_EVENT_ERROR) { + ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); ssl_handle_conn_origin_err(bev,ctx->mgr->logger); promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to original server failed."); } else if(events & BEV_EVENT_EOF) - { + { + ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); promise_failed(promise, FUTURE_ERROR_EXCEPTION, "original server closed."); } else if(events & BEV_EVENT_TIMEOUT) @@ -707,8 +789,11 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v } else if(events & BEV_EVENT_CONNECTED) { + + ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_NUM])); bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success + //The reference count of the SSL_SESSION is not incremented, so no need to free. ssl_sess = SSL_get0_session(s_stream->ssl); up_session_set(s_stream->mgr->up_sess_cache, (struct sockaddr *)&(ctx->addr), ctx->addrlen, s_stream->client_hello->sni, ssl_sess); @@ -724,6 +809,10 @@ static void peek_chello_on_succ(future_result_t * result, void * user) struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_get_ctx(p); struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. + if(chello->sni==NULL) + { + ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_SNI])); + } ctx->s_stream = ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL); ctx->bev = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_upstream, ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); @@ -741,7 +830,7 @@ static void peek_chello_on_fail(enum e_future_error err, const char * what, void { struct promise * p = (struct promise *) user; struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); - + ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO])); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream."); ssl_connect_origin_ctx_free(ctx); return; @@ -1095,9 +1184,11 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct struct promise * p = future_to_promise(f); promise_set_ctx(p, ctx, query_cert_ctx_free); ctx->is_origin_crt_vaild = ssl_conn_verify_cert(mgr->trust_CA_store, upstream->ssl); - + if(!ctx->is_origin_crt_vaild) + { + ATOMIC_INC(&(mgr->stat_val[SSL_FAKE_CRT])); + } ctx->f_query_cert = future_create(ask_keyring_on_succ, ask_keyring_on_fail, p); - //todo add a is_valid_cert flag to keyring manager query API. key_keeper_async_ask(ctx->f_query_cert, mgr->keeper_of_keys, keyring_id, ctx->origin_crt, ctx->is_origin_crt_vaild, evbase); return; @@ -1203,6 +1294,14 @@ retry: "Cannot create event. Closing fd.\n"); complete: + if(ctx->s_stream->dir=CONN_DIR_DOWNSTREAM) + { + ATOMIC_INC(&(ctx->s_stream->mgr->stat_val[SSL_DOWN_CLOSED])); + } + else + { + ATOMIC_INC(&(ctx->s_stream->mgr->stat_val[SSL_UP_CLOSED])); + } ssl_stream_free(ctx->s_stream); evutil_closesocket(fd); ssl_shutdown_ctx_free(ctx); @@ -1217,6 +1316,7 @@ complete: void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, evutil_socket_t fd) { struct ssl_shutdown_ctx * sslshutctx = NULL; + assert(fd==s_stream->_do_not_use.fd); sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); pxy_ssl_shutdown_cb(fd, 0, sslshutctx); } diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index b45a0be..9fb80d4 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -142,3 +142,7 @@ set_property(TARGET MESA_htable PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FR add_library(maatframe SHARED IMPORTED GLOBAL) set_property(TARGET maatframe PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libmaatframe.so) set_property(TARGET maatframe PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR}) + +add_library(MESA_field_stat SHARED IMPORTED GLOBAL) +set_property(TARGET MESA_field_stat PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libMESA_field_stat2.so) +set_property(TARGET MESA_field_stat PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR})