diff --git a/common/include/tfe_types.h b/common/include/tfe_types.h index 1633d53..b60a035 100644 --- a/common/include/tfe_types.h +++ b/common/include/tfe_types.h @@ -179,7 +179,7 @@ static inline char * tfe_stream_addr_to_str(const struct tfe_stream_addr * addr) inet_ntop(AF_INET6, &tuple4_v6->saddr, __src_addr, sizeof(__src_addr)); inet_ntop(AF_INET6, &tuple4_v6->daddr, __dst_addr, sizeof(__dst_addr)); - asprintf(&__str_ret, "%s %s %u %u", __src_addr, __dst_addr, __src_port, __dst_port); + asprintf(&__str_ret, "%s %u %s %u", __src_addr, __src_port, __dst_addr, __dst_port); } return __str_ret; diff --git a/common/include/tfe_utils.h b/common/include/tfe_utils.h index 1c3f3ac..19f950e 100644 --- a/common/include/tfe_utils.h +++ b/common/include/tfe_utils.h @@ -7,6 +7,7 @@ #include #define TFE_STRING_MAX 2048 +#define TFE_PATH_MAX 256 #define TFE_SYMBOL_MAX 64 #define TFE_THREAD_MAX 128 diff --git a/platform/include/internal/ssl_sess_cache.h b/platform/include/internal/ssl_sess_cache.h index 24aad61..e4cc601 100644 --- a/platform/include/internal/ssl_sess_cache.h +++ b/platform/include/internal/ssl_sess_cache.h @@ -14,3 +14,4 @@ SSL_SESSION * up_session_get(struct sess_cache * cache, struct sockaddr * addr, void down_session_set(struct sess_cache * cache, const SSL_SESSION * sess); void down_session_del(struct sess_cache * cache, const SSL_SESSION * sess); SSL_SESSION * down_session_get(struct sess_cache * cache, const unsigned char * id, int idlen); +void ssl_sess_cache_stat(struct sess_cache * cache, long long *size, long long* n_query,long long* n_hit); diff --git a/platform/src/key_keeper.cpp b/platform/src/key_keeper.cpp index dacb91d..48f2390 100644 --- a/platform/src/key_keeper.cpp +++ b/platform/src/key_keeper.cpp @@ -13,23 +13,30 @@ #define KEYRING_EXSITED 0 #define KEYRING_NOT_EXSITED -1 +enum key_keeper_mode{ + KK_MODE_NORMAL = 0, + KK_MODE_DEBUG, +}; + struct key_keeper { - char mode[TFE_STRING_MAX]; - char ca_path[TFE_STRING_MAX]; - char untrusted_ca_path[TFE_STRING_MAX]; - char cert_store_host[TFE_STRING_MAX]; + enum key_keeper_mode work_mode; + char trusted_ca_path[TFE_PATH_MAX]; + char untrusted_ca_path[TFE_PATH_MAX]; + char cert_store_host[TFE_SYMBOL_MAX]; unsigned int cert_store_port; unsigned int hash_slot_size; unsigned int hash_expire_seconds; MESA_htable_handle htable; void* logger; + X509* trusted_ca_cert; + EVP_PKEY* trusted_ca_key; + + X509* untrusted_ca_cert; + EVP_PKEY* untrusted_ca_key; }; -enum KEY_KEEPER_MODE{ - NORMAL = 0, - DEBUG, -}; + struct keyring_private { @@ -361,10 +368,9 @@ static long keyring_local_cache_query_cb(void * data, const uchar * key, uint si } -static struct keyring_private* generate_x509_keyring(X509* origin_cert, int keyring_id, const char* filename) +static struct keyring_private* generate_x509_keyring(X509* origin_cert, X509* ca, EVP_PKEY* cakey) { - X509* ca = ssl_x509_load(filename); - EVP_PKEY* cakey = ssl_key_load(filename); + //TODO: could be optimized to save cpu. EVP_PKEY* forge_key = ssl_key_genrsa(1024); X509* forge_cert = ssl_x509_forge(ca, cakey, origin_cert, forge_key, NULL, NULL); STACK_OF(X509)* chain = sk_X509_new_null(); @@ -376,8 +382,7 @@ static struct keyring_private* generate_x509_keyring(X509* origin_cert, int keyr keyring_set_key(_kyr, forge_key); keyring_set_cert(_kyr, forge_cert); keyring_set_chain(_kyr, chain); - X509_free(ca); - EVP_PKEY_free(cakey); + X509_free(forge_cert); EVP_PKEY_free(forge_key); sk_X509_pop_free(chain, X509_free); @@ -481,29 +486,71 @@ static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int return htable; } +void key_keeper_destroy(struct key_keeper *keeper) +{ + MESA_htable_destroy(keeper->htable, NULL); + X509_free(keeper->trusted_ca_cert); + EVP_PKEY_free(keeper->trusted_ca_key); + + X509_free(keeper->untrusted_ca_cert); + EVP_PKEY_free(keeper->untrusted_ca_key); + + free(keeper); + keeper = NULL; + return; +} + struct key_keeper* key_keeper_init(const char * profile, const char* section, void* logger) { struct key_keeper* keeper = ALLOC(struct key_keeper, 1); keeper->logger = logger; - MESA_load_profile_string_def(profile, section, "mode", keeper->mode, sizeof(keeper->mode), "debug"); - MESA_load_profile_string_def(profile, section, "ca_path", keeper->ca_path, sizeof(keeper->ca_path), "./conf/mesalab-ca.pem"); - MESA_load_profile_string_def(profile, section, "untrusted_ca_path", keeper->untrusted_ca_path, sizeof(keeper->untrusted_ca_path), "./conf/mesalab-ca.pem"); + char tmp[TFE_STRING_MAX]={0}; + MESA_load_profile_string_def(profile, section, "mode", tmp, sizeof(tmp), "debug"); + if(strncmp(tmp, "debug", TFE_STRING_MAX) == 0) + { + keeper->work_mode = KK_MODE_DEBUG; + } + else + { + keeper->work_mode = KK_MODE_NORMAL; + } + MESA_load_profile_string_def(profile, section, "ca_path", keeper->trusted_ca_path, sizeof(keeper->trusted_ca_path), "./conf/mesalab-ca.pem"); + MESA_load_profile_string_def(profile, section, "untrusted_ca_path", keeper->untrusted_ca_path, sizeof(keeper->untrusted_ca_path), "./conf/mesalab-ca-untrust.pem"); MESA_load_profile_string_def(profile, section, "cert_store_host", keeper->cert_store_host, sizeof(keeper->cert_store_host), "xxxxx"); MESA_load_profile_uint_def(profile, section, "cert_store_port", &(keeper->cert_store_port), 80); MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024*128); MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5*60); keeper->htable = create_hash_table(keeper->hash_slot_size, keeper->hash_expire_seconds); + if(0==strcmp(keeper->untrusted_ca_path, keeper->trusted_ca_path)) + { + TFE_LOG_ERROR(logger, "Warnning: Trusted and Untrusted Root CA share the same path % .", keeper->trusted_ca_path); + } + if(keeper->work_mode==KK_MODE_DEBUG) + { + keeper->trusted_ca_cert=ssl_x509_load(keeper->trusted_ca_path); + keeper->trusted_ca_key=ssl_key_load(keeper->trusted_ca_path); + if(keeper->trusted_ca_cert==NULL||keeper->trusted_ca_key==NULL) + { + TFE_LOG_ERROR(logger, "Load Trusted Root CA %s failed.", keeper->trusted_ca_path); + goto error_out; + } + keeper->untrusted_ca_cert=ssl_x509_load(keeper->untrusted_ca_path); + keeper->untrusted_ca_key=ssl_key_load(keeper->untrusted_ca_path); + if(keeper->untrusted_ca_cert==NULL||keeper->trusted_ca_key==NULL) + { + TFE_LOG_ERROR(logger, "Load Untrusted Root CA %s failed.", keeper->untrusted_ca_path); + goto error_out; + } + } TFE_LOG_INFO(logger, "MESA_load_profile, [%s]: mode:%s, ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d", - section, keeper->mode, keeper->ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds); + section, tmp, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds); + return keeper; -} - -void key_keeper_destroy(struct key_keeper *keeper) -{ - MESA_htable_destroy(keeper->htable, NULL); - free(keeper); - keeper = NULL; - return; + +error_out: + key_keeper_destroy(keeper); + return NULL; + } struct keyring* key_keeper_release_keyring(future_result_t* result) @@ -573,13 +620,9 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c //printf("KEYRING_EXSITED\n"); return; } - int mode = 0; - if(strncmp(keeper->mode, "debug", TFE_STRING_MAX) == 0) + switch(keeper->work_mode) { - mode = 1; - } - switch(mode){ - case NORMAL: + case KK_MODE_NORMAL: { char* origin_cert_pem = transform_cert_to_pem(origin_cert); if(origin_cert_pem == NULL) @@ -614,18 +657,17 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c free(url); break; } - case DEBUG: + case KK_MODE_DEBUG: { - char* filename = NULL; + struct keyring_private* kyr=NULL; if(is_cert_valid == 1) { - filename = keeper->ca_path; + kyr=generate_x509_keyring(origin_cert, keeper->trusted_ca_cert, keeper->trusted_ca_key); } else { - filename = keeper->untrusted_ca_path; + kyr=generate_x509_keyring(origin_cert, keeper->untrusted_ca_cert, keeper->untrusted_ca_key); } - struct keyring_private* kyr = generate_x509_keyring(origin_cert, keyring_id, filename); if(kyr) { keyring_ref_inc(kyr); diff --git a/platform/src/ssl_sess_cache.cpp b/platform/src/ssl_sess_cache.cpp index 73c9821..93cf070 100644 --- a/platform/src/ssl_sess_cache.cpp +++ b/platform/src/ssl_sess_cache.cpp @@ -146,42 +146,33 @@ static long sess_cache_set_cb(void * data, const uchar * key, uint size, void * } } -static size_t upsess_mk_key(struct sockaddr * addr, socklen_t addrlen, const char * sni, unsigned char ** key_buf) +static size_t upsess_mk_key(struct sockaddr * res, socklen_t addrlen, const char * sni, unsigned char ** key_buf) { size_t key_size = 0; - unsigned char * tmp = NULL; - size_t tmp_size; - short port; - size_t snilen; - - switch (addr->sa_family) - { - case AF_INET: - tmp = (unsigned char *)&((struct sockaddr_in *) addr)->sin_addr; - tmp_size = sizeof(struct in_addr); - port = ((struct sockaddr_in *) addr)->sin_port; + unsigned short port; + + char *s = NULL; + switch(res->sa_family) { + case AF_INET: { + struct sockaddr_in *addr_in = (struct sockaddr_in *)res; + s = (char*) malloc(INET_ADDRSTRLEN); + inet_ntop(AF_INET, &(addr_in->sin_addr), s, INET_ADDRSTRLEN); + port = ntohs(addr_in->sin_port); break; - case AF_INET6: - tmp = (unsigned char *)&((struct sockaddr_in6 *) addr)->sin6_addr; - tmp_size = sizeof(struct in6_addr); - port = ((struct sockaddr_in6 *) addr)->sin6_port; + } + case AF_INET6: { + struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)res; + s = (char*) malloc(INET6_ADDRSTRLEN); + inet_ntop(AF_INET6, &(addr_in6->sin6_addr), s, INET6_ADDRSTRLEN); + port = ntohs(addr_in6->sin6_port); break; + } default: - //should never happens. - assert(0); break; } - - snilen = sni ? strlen(sni) : 0; - key_size = tmp_size + sizeof(port) + snilen; - - *key_buf = ALLOC(unsigned char, key_size); - unsigned char * p = *key_buf; - - memcpy(p, tmp, tmp_size); - p += tmp_size; - memcpy(p, (char *) &port, sizeof(port)); - p += sizeof(port); + + key_size=asprintf((char**)key_buf,"%s:%u:%s",s, port, sni); + free(s); return key_size; } @@ -199,6 +190,7 @@ void up_session_set(struct sess_cache * cache, struct sockaddr * addr, socklen_t struct sess_set_args set_args={.hash = cache->hash, .new_sess = asn1}; MESA_htable_search_cb(cache->hash, key, key_size, sess_cache_set_cb, &set_args, &cb_ret); +// printf("%s %s\n", __FUNCTION__, key); if (cb_ret == SESS_CACHE_UPDATE_OLD) { ssl_sess_free_serialized(asn1); @@ -218,6 +210,7 @@ SSL_SESSION * up_session_get(struct sess_cache * cache, struct sockaddr * addr, unsigned char * key = NULL; key_size = upsess_mk_key(addr, addr_len, sni, &key); MESA_htable_search_cb(cache->hash, key, key_size, sess_cache_get_cb, &sess, &cb_ret); +// printf("%s %s\n", __FUNCTION__, key); free(key); key = NULL; if (cb_ret == SESS_CACHE_FOUND) @@ -338,4 +331,11 @@ void ssl_sess_cache_destroy(struct sess_cache * cache) free(cache); return; } +void ssl_sess_cache_stat(struct sess_cache * cache, long long *size, long long *n_query, long long* n_hit) +{ + *size=MESA_htable_get_elem_num(cache->hash); + *n_hit=cache->hit_cnt; + *n_query=cache->hit_cnt+cache->miss_cnt; + return; +} diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index 0cf2da7..201b25b 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -49,16 +49,24 @@ enum ssl_stream_stat { - SSL_UP_NUM, + SSL_UP_NEW, SSL_UP_ERR, SSL_UP_CLOSING, - SSL_UP_DIRTY_CLOSED, SSL_UP_CLOSED, - SSL_DOWN_NUM, + SSL_UP_DIRTY_CLOSED, + SSL_UP_CACHE_SZ, + SSL_UP_CACHE_QUERY, + SSL_UP_CACHE_HIT, + + SSL_DOWN_NEW, SSL_DOWN_ERR, SSL_DOWN_CLOSING, SSL_DOWN_CLOSED, SSL_DOWN_DIRTY_CLOSED, + SSL_DOWN_CACHE_SZ, + SSL_DOWN_CACHE_QUERY, + SSL_DOWN_CACHE_HIT, + SSL_NO_CHELLO, SSL_NO_SNI, SSL_FAKE_CRT, @@ -168,6 +176,8 @@ struct ssl_shutdown_ctx struct ssl_stream * s_stream; struct event_base * evbase; struct event * ev; + struct ssl_mgr* mgr; + enum tfe_conn_dir dir; unsigned int retries; }; struct fs_spec @@ -179,16 +189,24 @@ void ssl_stat_init(struct ssl_mgr * mgr) { int i=0; const char* spec[SSL_STAT_MAX]; - spec[SSL_UP_NUM]="ssl_up"; - spec[SSL_UP_ERR]="sslu_err"; - spec[SSL_UP_CLOSING]="sslu_clsing"; - spec[SSL_UP_CLOSED]="sslu_clsd"; - spec[SSL_UP_DIRTY_CLOSED]="sslu_dirty_cls"; - spec[SSL_DOWN_NUM]="ssl_down"; - spec[SSL_DOWN_ERR]="ssld_err"; - spec[SSL_DOWN_CLOSING]="ssld_clsing"; + spec[SSL_UP_NEW]="ussl_new"; + spec[SSL_UP_ERR]="ussl_err"; + spec[SSL_UP_CLOSING]="ussl_clsing"; + spec[SSL_UP_CLOSED]="ussl_clsed"; + spec[SSL_UP_DIRTY_CLOSED]="ussl_dirty_cls"; + spec[SSL_UP_CACHE_SZ]="usess_cache"; + spec[SSL_UP_CACHE_QUERY]="usess_query"; + spec[SSL_UP_CACHE_HIT]="usess_hitcnt"; + + spec[SSL_DOWN_NEW]="dssl_new"; + spec[SSL_DOWN_ERR]="dssl_err"; + spec[SSL_DOWN_CLOSING]="dssl_clsing"; + spec[SSL_DOWN_CLOSED]="dssl_clsed"; spec[SSL_DOWN_DIRTY_CLOSED]="ssld_dirty_cls"; - spec[SSL_DOWN_CLOSED]="ssld_clsd"; + spec[SSL_DOWN_CACHE_SZ]="dsess_cache"; + spec[SSL_DOWN_CACHE_QUERY]="dcache_query"; + spec[SSL_DOWN_CACHE_HIT]="dsess_hitcnt"; + spec[SSL_NO_CHELLO]="ssl_no_chlo"; spec[SSL_NO_SNI]="ssl_no_sni"; spec[SSL_FAKE_CRT]="ssl_fk_crt"; @@ -197,6 +215,31 @@ void ssl_stat_init(struct ssl_mgr * mgr) { mgr->fs_id[i]=FS_register(mgr->fs_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,spec[i]); } + + int value=mgr->fs_id[SSL_UP_CACHE_HIT]; + FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); + value=mgr->fs_id[SSL_UP_CACHE_QUERY]; + FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); + + FS_register_ratio(mgr->fs_handle, + mgr->fs_id[SSL_UP_CACHE_HIT], + mgr->fs_id[SSL_UP_CACHE_QUERY], + 1, + FS_STYLE_STATUS, + FS_CALC_CURRENT, + "usess_hit"); + value=mgr->fs_id[SSL_DOWN_CACHE_HIT]; + FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); + value=mgr->fs_id[SSL_DOWN_CACHE_QUERY]; + FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); + + FS_register_ratio(mgr->fs_handle, + mgr->fs_id[SSL_DOWN_CACHE_HIT], + mgr->fs_id[SSL_DOWN_CACHE_QUERY], + 1, + FS_STYLE_STATUS, + FS_CALC_CURRENT, + "dsess_hit"); return; } static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt); @@ -223,11 +266,13 @@ struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enu assert(ret == 0); switch (dir) { - case CONN_DIR_DOWNSTREAM: + case CONN_DIR_DOWNSTREAM: + ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_NEW])); s_stream->ssl = downstream_ssl_create(mgr, kyr); s_stream->keyring = kyr; break; case CONN_DIR_UPSTREAM: + ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_NEW])); s_stream->ssl = upstream_ssl_create(mgr, client_hello, fd); s_stream->client_hello = client_hello; break; @@ -247,7 +292,8 @@ static void ssl_stream_free(struct ssl_stream * s_stream) { key_keeper_free_keyring(s_stream->keyring); s_stream->keyring = NULL; - } + } + ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED])); break; case CONN_DIR_UPSTREAM: if (s_stream->client_hello != NULL) @@ -255,6 +301,7 @@ static void ssl_stream_free(struct ssl_stream * s_stream) ssl_chello_free(s_stream->client_hello); s_stream->client_hello = NULL; } + ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_CLOSED])); break; default: assert(0); } @@ -337,6 +384,8 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) { struct ssl_mgr *mgr=(struct ssl_mgr *)arg; int i=0; + ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT])); + ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT])); for(i=0;ifs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i]))); @@ -806,12 +855,11 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v } else if(events & BEV_EVENT_TIMEOUT) { + ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); promise_failed(promise, FUTURE_ERROR_TIMEOUT, NULL); } else if(events & BEV_EVENT_CONNECTED) - { - - ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_NUM])); + { bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success //The reference count of the SSL_SESSION is not incremented, so no need to free. @@ -865,7 +913,7 @@ extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, e int ret = 0; ctx->addrlen = sizeof(ctx->addr); - ret = getpeername(fd_downstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen)); + ret = getpeername(fd_upstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen)); assert(ret == 0); ctx->fd_downstream = fd_downstream; @@ -1229,12 +1277,20 @@ static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stre ctx->evbase = evbase; ctx->s_stream = s_stream; ctx->ev = NULL; + ctx->mgr = s_stream->mgr; + ctx->dir = s_stream->dir; ctx->retries = 0; + ctx->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_INC(&(ctx->mgr->stat_val[SSL_DOWN_CLOSING])) + : ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_CLOSING])); + return ctx; } static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx) { + ctx->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_DEC(&(ctx->mgr->stat_val[SSL_DOWN_CLOSING])) + : ATOMIC_DEC(&(ctx->mgr->stat_val[SSL_UP_CLOSING])); + memset(ctx, 0, sizeof(struct ssl_shutdown_ctx)); free(ctx); } @@ -1332,17 +1388,7 @@ retry: } return; complete: - if(ctx->s_stream->dir==CONN_DIR_DOWNSTREAM) - { - ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_CLOSED])); - ATOMIC_DEC(&(mgr->stat_val[SSL_DOWN_CLOSING])); - } - else - { - ATOMIC_INC(&(mgr->stat_val[SSL_UP_CLOSED])); - ATOMIC_DEC(&(mgr->stat_val[SSL_UP_CLOSING])); - } - + ssl_stream_free(ctx->s_stream); evutil_closesocket(fd); ssl_shutdown_ctx_free(ctx); @@ -1359,7 +1405,5 @@ void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_bas struct ssl_shutdown_ctx * sslshutctx = NULL; assert(fd==s_stream->_do_not_use.fd); sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase); - s_stream->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSING])) - : ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_CLOSING])); pxy_ssl_shutdown_cb(fd, 0, sslshutctx); }