ssl stream修复upstream session cache key生成错误的bug。

This commit is contained in:
zhengchao
2018-10-05 18:30:58 +08:00
parent 9ec072343b
commit d7bf70b4b0
6 changed files with 185 additions and 97 deletions

View File

@@ -179,7 +179,7 @@ static inline char * tfe_stream_addr_to_str(const struct tfe_stream_addr * addr)
inet_ntop(AF_INET6, &tuple4_v6->saddr, __src_addr, sizeof(__src_addr));
inet_ntop(AF_INET6, &tuple4_v6->daddr, __dst_addr, sizeof(__dst_addr));
asprintf(&__str_ret, "%s %s %u %u", __src_addr, __dst_addr, __src_port, __dst_port);
asprintf(&__str_ret, "%s %u %s %u", __src_addr, __src_port, __dst_addr, __dst_port);
}
return __str_ret;

View File

@@ -7,6 +7,7 @@
#include <MESA/MESA_handle_logger.h>
#define TFE_STRING_MAX 2048
#define TFE_PATH_MAX 256
#define TFE_SYMBOL_MAX 64
#define TFE_THREAD_MAX 128

View File

@@ -14,3 +14,4 @@ SSL_SESSION * up_session_get(struct sess_cache * cache, struct sockaddr * addr,
void down_session_set(struct sess_cache * cache, const SSL_SESSION * sess);
void down_session_del(struct sess_cache * cache, const SSL_SESSION * sess);
SSL_SESSION * down_session_get(struct sess_cache * cache, const unsigned char * id, int idlen);
void ssl_sess_cache_stat(struct sess_cache * cache, long long *size, long long* n_query,long long* n_hit);

View File

@@ -13,23 +13,30 @@
#define KEYRING_EXSITED 0
#define KEYRING_NOT_EXSITED -1
enum key_keeper_mode{
KK_MODE_NORMAL = 0,
KK_MODE_DEBUG,
};
struct key_keeper
{
char mode[TFE_STRING_MAX];
char ca_path[TFE_STRING_MAX];
char untrusted_ca_path[TFE_STRING_MAX];
char cert_store_host[TFE_STRING_MAX];
enum key_keeper_mode work_mode;
char trusted_ca_path[TFE_PATH_MAX];
char untrusted_ca_path[TFE_PATH_MAX];
char cert_store_host[TFE_SYMBOL_MAX];
unsigned int cert_store_port;
unsigned int hash_slot_size;
unsigned int hash_expire_seconds;
MESA_htable_handle htable;
void* logger;
X509* trusted_ca_cert;
EVP_PKEY* trusted_ca_key;
X509* untrusted_ca_cert;
EVP_PKEY* untrusted_ca_key;
};
enum KEY_KEEPER_MODE{
NORMAL = 0,
DEBUG,
};
struct keyring_private
{
@@ -361,10 +368,9 @@ static long keyring_local_cache_query_cb(void * data, const uchar * key, uint si
}
static struct keyring_private* generate_x509_keyring(X509* origin_cert, int keyring_id, const char* filename)
static struct keyring_private* generate_x509_keyring(X509* origin_cert, X509* ca, EVP_PKEY* cakey)
{
X509* ca = ssl_x509_load(filename);
EVP_PKEY* cakey = ssl_key_load(filename);
//TODO: could be optimized to save cpu.
EVP_PKEY* forge_key = ssl_key_genrsa(1024);
X509* forge_cert = ssl_x509_forge(ca, cakey, origin_cert, forge_key, NULL, NULL);
STACK_OF(X509)* chain = sk_X509_new_null();
@@ -376,8 +382,7 @@ static struct keyring_private* generate_x509_keyring(X509* origin_cert, int keyr
keyring_set_key(_kyr, forge_key);
keyring_set_cert(_kyr, forge_cert);
keyring_set_chain(_kyr, chain);
X509_free(ca);
EVP_PKEY_free(cakey);
X509_free(forge_cert);
EVP_PKEY_free(forge_key);
sk_X509_pop_free(chain, X509_free);
@@ -481,29 +486,71 @@ static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int
return htable;
}
void key_keeper_destroy(struct key_keeper *keeper)
{
MESA_htable_destroy(keeper->htable, NULL);
X509_free(keeper->trusted_ca_cert);
EVP_PKEY_free(keeper->trusted_ca_key);
X509_free(keeper->untrusted_ca_cert);
EVP_PKEY_free(keeper->untrusted_ca_key);
free(keeper);
keeper = NULL;
return;
}
struct key_keeper* key_keeper_init(const char * profile, const char* section, void* logger)
{
struct key_keeper* keeper = ALLOC(struct key_keeper, 1);
keeper->logger = logger;
MESA_load_profile_string_def(profile, section, "mode", keeper->mode, sizeof(keeper->mode), "debug");
MESA_load_profile_string_def(profile, section, "ca_path", keeper->ca_path, sizeof(keeper->ca_path), "./conf/mesalab-ca.pem");
MESA_load_profile_string_def(profile, section, "untrusted_ca_path", keeper->untrusted_ca_path, sizeof(keeper->untrusted_ca_path), "./conf/mesalab-ca.pem");
char tmp[TFE_STRING_MAX]={0};
MESA_load_profile_string_def(profile, section, "mode", tmp, sizeof(tmp), "debug");
if(strncmp(tmp, "debug", TFE_STRING_MAX) == 0)
{
keeper->work_mode = KK_MODE_DEBUG;
}
else
{
keeper->work_mode = KK_MODE_NORMAL;
}
MESA_load_profile_string_def(profile, section, "ca_path", keeper->trusted_ca_path, sizeof(keeper->trusted_ca_path), "./conf/mesalab-ca.pem");
MESA_load_profile_string_def(profile, section, "untrusted_ca_path", keeper->untrusted_ca_path, sizeof(keeper->untrusted_ca_path), "./conf/mesalab-ca-untrust.pem");
MESA_load_profile_string_def(profile, section, "cert_store_host", keeper->cert_store_host, sizeof(keeper->cert_store_host), "xxxxx");
MESA_load_profile_uint_def(profile, section, "cert_store_port", &(keeper->cert_store_port), 80);
MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024*128);
MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5*60);
keeper->htable = create_hash_table(keeper->hash_slot_size, keeper->hash_expire_seconds);
if(0==strcmp(keeper->untrusted_ca_path, keeper->trusted_ca_path))
{
TFE_LOG_ERROR(logger, "Warnning: Trusted and Untrusted Root CA share the same path % .", keeper->trusted_ca_path);
}
if(keeper->work_mode==KK_MODE_DEBUG)
{
keeper->trusted_ca_cert=ssl_x509_load(keeper->trusted_ca_path);
keeper->trusted_ca_key=ssl_key_load(keeper->trusted_ca_path);
if(keeper->trusted_ca_cert==NULL||keeper->trusted_ca_key==NULL)
{
TFE_LOG_ERROR(logger, "Load Trusted Root CA %s failed.", keeper->trusted_ca_path);
goto error_out;
}
keeper->untrusted_ca_cert=ssl_x509_load(keeper->untrusted_ca_path);
keeper->untrusted_ca_key=ssl_key_load(keeper->untrusted_ca_path);
if(keeper->untrusted_ca_cert==NULL||keeper->trusted_ca_key==NULL)
{
TFE_LOG_ERROR(logger, "Load Untrusted Root CA %s failed.", keeper->untrusted_ca_path);
goto error_out;
}
}
TFE_LOG_INFO(logger, "MESA_load_profile, [%s]: mode:%s, ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d",
section, keeper->mode, keeper->ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds);
section, tmp, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds);
return keeper;
}
void key_keeper_destroy(struct key_keeper *keeper)
{
MESA_htable_destroy(keeper->htable, NULL);
free(keeper);
keeper = NULL;
return;
error_out:
key_keeper_destroy(keeper);
return NULL;
}
struct keyring* key_keeper_release_keyring(future_result_t* result)
@@ -573,13 +620,9 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c
//printf("KEYRING_EXSITED\n");
return;
}
int mode = 0;
if(strncmp(keeper->mode, "debug", TFE_STRING_MAX) == 0)
switch(keeper->work_mode)
{
mode = 1;
}
switch(mode){
case NORMAL:
case KK_MODE_NORMAL:
{
char* origin_cert_pem = transform_cert_to_pem(origin_cert);
if(origin_cert_pem == NULL)
@@ -614,18 +657,17 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c
free(url);
break;
}
case DEBUG:
case KK_MODE_DEBUG:
{
char* filename = NULL;
struct keyring_private* kyr=NULL;
if(is_cert_valid == 1)
{
filename = keeper->ca_path;
kyr=generate_x509_keyring(origin_cert, keeper->trusted_ca_cert, keeper->trusted_ca_key);
}
else
{
filename = keeper->untrusted_ca_path;
kyr=generate_x509_keyring(origin_cert, keeper->untrusted_ca_cert, keeper->untrusted_ca_key);
}
struct keyring_private* kyr = generate_x509_keyring(origin_cert, keyring_id, filename);
if(kyr)
{
keyring_ref_inc(kyr);

View File

@@ -146,42 +146,33 @@ static long sess_cache_set_cb(void * data, const uchar * key, uint size, void *
}
}
static size_t upsess_mk_key(struct sockaddr * addr, socklen_t addrlen, const char * sni, unsigned char ** key_buf)
static size_t upsess_mk_key(struct sockaddr * res, socklen_t addrlen, const char * sni, unsigned char ** key_buf)
{
size_t key_size = 0;
unsigned char * tmp = NULL;
size_t tmp_size;
short port;
size_t snilen;
switch (addr->sa_family)
{
case AF_INET:
tmp = (unsigned char *)&((struct sockaddr_in *) addr)->sin_addr;
tmp_size = sizeof(struct in_addr);
port = ((struct sockaddr_in *) addr)->sin_port;
unsigned short port;
char *s = NULL;
switch(res->sa_family) {
case AF_INET: {
struct sockaddr_in *addr_in = (struct sockaddr_in *)res;
s = (char*) malloc(INET_ADDRSTRLEN);
inet_ntop(AF_INET, &(addr_in->sin_addr), s, INET_ADDRSTRLEN);
port = ntohs(addr_in->sin_port);
break;
case AF_INET6:
tmp = (unsigned char *)&((struct sockaddr_in6 *) addr)->sin6_addr;
tmp_size = sizeof(struct in6_addr);
port = ((struct sockaddr_in6 *) addr)->sin6_port;
}
case AF_INET6: {
struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)res;
s = (char*) malloc(INET6_ADDRSTRLEN);
inet_ntop(AF_INET6, &(addr_in6->sin6_addr), s, INET6_ADDRSTRLEN);
port = ntohs(addr_in6->sin6_port);
break;
}
default:
//should never happens.
assert(0);
break;
}
snilen = sni ? strlen(sni) : 0;
key_size = tmp_size + sizeof(port) + snilen;
*key_buf = ALLOC(unsigned char, key_size);
unsigned char * p = *key_buf;
memcpy(p, tmp, tmp_size);
p += tmp_size;
memcpy(p, (char *) &port, sizeof(port));
p += sizeof(port);
key_size=asprintf((char**)key_buf,"%s:%u:%s",s, port, sni);
free(s);
return key_size;
}
@@ -199,6 +190,7 @@ void up_session_set(struct sess_cache * cache, struct sockaddr * addr, socklen_t
struct sess_set_args set_args={.hash = cache->hash, .new_sess = asn1};
MESA_htable_search_cb(cache->hash, key, key_size, sess_cache_set_cb, &set_args, &cb_ret);
// printf("%s %s\n", __FUNCTION__, key);
if (cb_ret == SESS_CACHE_UPDATE_OLD)
{
ssl_sess_free_serialized(asn1);
@@ -218,6 +210,7 @@ SSL_SESSION * up_session_get(struct sess_cache * cache, struct sockaddr * addr,
unsigned char * key = NULL;
key_size = upsess_mk_key(addr, addr_len, sni, &key);
MESA_htable_search_cb(cache->hash, key, key_size, sess_cache_get_cb, &sess, &cb_ret);
// printf("%s %s\n", __FUNCTION__, key);
free(key);
key = NULL;
if (cb_ret == SESS_CACHE_FOUND)
@@ -338,4 +331,11 @@ void ssl_sess_cache_destroy(struct sess_cache * cache)
free(cache);
return;
}
void ssl_sess_cache_stat(struct sess_cache * cache, long long *size, long long *n_query, long long* n_hit)
{
*size=MESA_htable_get_elem_num(cache->hash);
*n_hit=cache->hit_cnt;
*n_query=cache->hit_cnt+cache->miss_cnt;
return;
}

View File

@@ -49,16 +49,24 @@
enum ssl_stream_stat
{
SSL_UP_NUM,
SSL_UP_NEW,
SSL_UP_ERR,
SSL_UP_CLOSING,
SSL_UP_DIRTY_CLOSED,
SSL_UP_CLOSED,
SSL_DOWN_NUM,
SSL_UP_DIRTY_CLOSED,
SSL_UP_CACHE_SZ,
SSL_UP_CACHE_QUERY,
SSL_UP_CACHE_HIT,
SSL_DOWN_NEW,
SSL_DOWN_ERR,
SSL_DOWN_CLOSING,
SSL_DOWN_CLOSED,
SSL_DOWN_DIRTY_CLOSED,
SSL_DOWN_CACHE_SZ,
SSL_DOWN_CACHE_QUERY,
SSL_DOWN_CACHE_HIT,
SSL_NO_CHELLO,
SSL_NO_SNI,
SSL_FAKE_CRT,
@@ -168,6 +176,8 @@ struct ssl_shutdown_ctx
struct ssl_stream * s_stream;
struct event_base * evbase;
struct event * ev;
struct ssl_mgr* mgr;
enum tfe_conn_dir dir;
unsigned int retries;
};
struct fs_spec
@@ -179,16 +189,24 @@ void ssl_stat_init(struct ssl_mgr * mgr)
{
int i=0;
const char* spec[SSL_STAT_MAX];
spec[SSL_UP_NUM]="ssl_up";
spec[SSL_UP_ERR]="sslu_err";
spec[SSL_UP_CLOSING]="sslu_clsing";
spec[SSL_UP_CLOSED]="sslu_clsd";
spec[SSL_UP_DIRTY_CLOSED]="sslu_dirty_cls";
spec[SSL_DOWN_NUM]="ssl_down";
spec[SSL_DOWN_ERR]="ssld_err";
spec[SSL_DOWN_CLOSING]="ssld_clsing";
spec[SSL_UP_NEW]="ussl_new";
spec[SSL_UP_ERR]="ussl_err";
spec[SSL_UP_CLOSING]="ussl_clsing";
spec[SSL_UP_CLOSED]="ussl_clsed";
spec[SSL_UP_DIRTY_CLOSED]="ussl_dirty_cls";
spec[SSL_UP_CACHE_SZ]="usess_cache";
spec[SSL_UP_CACHE_QUERY]="usess_query";
spec[SSL_UP_CACHE_HIT]="usess_hitcnt";
spec[SSL_DOWN_NEW]="dssl_new";
spec[SSL_DOWN_ERR]="dssl_err";
spec[SSL_DOWN_CLOSING]="dssl_clsing";
spec[SSL_DOWN_CLOSED]="dssl_clsed";
spec[SSL_DOWN_DIRTY_CLOSED]="ssld_dirty_cls";
spec[SSL_DOWN_CLOSED]="ssld_clsd";
spec[SSL_DOWN_CACHE_SZ]="dsess_cache";
spec[SSL_DOWN_CACHE_QUERY]="dcache_query";
spec[SSL_DOWN_CACHE_HIT]="dsess_hitcnt";
spec[SSL_NO_CHELLO]="ssl_no_chlo";
spec[SSL_NO_SNI]="ssl_no_sni";
spec[SSL_FAKE_CRT]="ssl_fk_crt";
@@ -197,6 +215,31 @@ void ssl_stat_init(struct ssl_mgr * mgr)
{
mgr->fs_id[i]=FS_register(mgr->fs_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,spec[i]);
}
int value=mgr->fs_id[SSL_UP_CACHE_HIT];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
value=mgr->fs_id[SSL_UP_CACHE_QUERY];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(mgr->fs_handle,
mgr->fs_id[SSL_UP_CACHE_HIT],
mgr->fs_id[SSL_UP_CACHE_QUERY],
1,
FS_STYLE_STATUS,
FS_CALC_CURRENT,
"usess_hit");
value=mgr->fs_id[SSL_DOWN_CACHE_HIT];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
value=mgr->fs_id[SSL_DOWN_CACHE_QUERY];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(mgr->fs_handle,
mgr->fs_id[SSL_DOWN_CACHE_HIT],
mgr->fs_id[SSL_DOWN_CACHE_QUERY],
1,
FS_STYLE_STATUS,
FS_CALC_CURRENT,
"dsess_hit");
return;
}
static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt);
@@ -223,11 +266,13 @@ struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enu
assert(ret == 0);
switch (dir)
{
case CONN_DIR_DOWNSTREAM:
case CONN_DIR_DOWNSTREAM:
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_NEW]));
s_stream->ssl = downstream_ssl_create(mgr, kyr);
s_stream->keyring = kyr;
break;
case CONN_DIR_UPSTREAM:
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_NEW]));
s_stream->ssl = upstream_ssl_create(mgr, client_hello, fd);
s_stream->client_hello = client_hello;
break;
@@ -247,7 +292,8 @@ static void ssl_stream_free(struct ssl_stream * s_stream)
{
key_keeper_free_keyring(s_stream->keyring);
s_stream->keyring = NULL;
}
}
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED]));
break;
case CONN_DIR_UPSTREAM:
if (s_stream->client_hello != NULL)
@@ -255,6 +301,7 @@ static void ssl_stream_free(struct ssl_stream * s_stream)
ssl_chello_free(s_stream->client_hello);
s_stream->client_hello = NULL;
}
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_CLOSED]));
break;
default: assert(0);
}
@@ -337,6 +384,8 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
{
struct ssl_mgr *mgr=(struct ssl_mgr *)arg;
int i=0;
ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT]));
ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT]));
for(i=0;i<SSL_STAT_MAX;i++)
{
FS_operate(mgr->fs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i])));
@@ -806,12 +855,11 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v
}
else if(events & BEV_EVENT_TIMEOUT)
{
ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
promise_failed(promise, FUTURE_ERROR_TIMEOUT, NULL);
}
else if(events & BEV_EVENT_CONNECTED)
{
ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_NUM]));
{
bufferevent_disable(ctx->bev, EV_READ | EV_WRITE);
bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success
//The reference count of the SSL_SESSION is not incremented, so no need to free.
@@ -865,7 +913,7 @@ extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, e
int ret = 0;
ctx->addrlen = sizeof(ctx->addr);
ret = getpeername(fd_downstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen));
ret = getpeername(fd_upstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen));
assert(ret == 0);
ctx->fd_downstream = fd_downstream;
@@ -1229,12 +1277,20 @@ static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stre
ctx->evbase = evbase;
ctx->s_stream = s_stream;
ctx->ev = NULL;
ctx->mgr = s_stream->mgr;
ctx->dir = s_stream->dir;
ctx->retries = 0;
ctx->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_INC(&(ctx->mgr->stat_val[SSL_DOWN_CLOSING]))
: ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_CLOSING]));
return ctx;
}
static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx)
{
ctx->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_DEC(&(ctx->mgr->stat_val[SSL_DOWN_CLOSING]))
: ATOMIC_DEC(&(ctx->mgr->stat_val[SSL_UP_CLOSING]));
memset(ctx, 0, sizeof(struct ssl_shutdown_ctx));
free(ctx);
}
@@ -1332,17 +1388,7 @@ retry:
}
return;
complete:
if(ctx->s_stream->dir==CONN_DIR_DOWNSTREAM)
{
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_CLOSED]));
ATOMIC_DEC(&(mgr->stat_val[SSL_DOWN_CLOSING]));
}
else
{
ATOMIC_INC(&(mgr->stat_val[SSL_UP_CLOSED]));
ATOMIC_DEC(&(mgr->stat_val[SSL_UP_CLOSING]));
}
ssl_stream_free(ctx->s_stream);
evutil_closesocket(fd);
ssl_shutdown_ctx_free(ctx);
@@ -1359,7 +1405,5 @@ void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_bas
struct ssl_shutdown_ctx * sslshutctx = NULL;
assert(fd==s_stream->_do_not_use.fd);
sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase);
s_stream->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSING]))
: ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_CLOSING]));
pxy_ssl_shutdown_cb(fd, 0, sslshutctx);
}