gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-2993 将 TFE key keeper 哈希表的 pthread_mutex 改为 pthread_rwlock

Browse Source 
* 插入新证书时若哈希表满，则删除 10% 最老的证书
	* 查询到结果之后再判断是否超时，若超时则删除，并重新查询 certstore
This commit is contained in:
luwenpeng
2020-08-31 13:11:27 +08:00
parent fb02543749
commit 1e5b2ceaed
2 changed files with 106 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
platform/src/key_keeper.cpp
View File

@@ -23,6 +23,7 @@
#define HTABLE_MAX_KEY_LEN 256 #define HTABLE_MAX_KEY_LEN 256
#define KEYRING_EXSITED 0 #define KEYRING_EXSITED 0
#define KEYRING_NOT_EXSITED -1 #define KEYRING_NOT_EXSITED -1
#define KEYRING_EXPIRE -2
enum key_keeper_mode{ enum key_keeper_mode{
KK_MODE_CERT_STORE = 0, KK_MODE_CERT_STORE = 0,
@@ -38,6 +39,8 @@ struct key_keeper
unsigned int cert_store_port; unsigned int cert_store_port;
unsigned int hash_slot_size; unsigned int hash_slot_size;
unsigned int hash_expire_seconds; unsigned int hash_expire_seconds;
unsigned int del_elem_num_once_when_full;
unsigned int max_elem_num;
MESA_htable_handle cert_cache; MESA_htable_handle cert_cache;
void* logger; void* logger;
X509* trusted_ca_cert; X509* trusted_ca_cert;
@@ -51,6 +54,7 @@ struct key_keeper
int cert_expire_time; int cert_expire_time;
unsigned int enable_health_check; unsigned int enable_health_check;
pthread_t thread; pthread_t thread;
pthread_rwlock_t rwlock;
}; };
@@ -60,6 +64,7 @@ struct keyring_private
struct keyring head; struct keyring head;
pthread_mutex_t mutex; pthread_mutex_t mutex;
size_t references; size_t references;
time_t update_time;
}; };
struct key_keeper_promise_ctx struct key_keeper_promise_ctx
@@ -336,10 +341,17 @@ static long keyring_local_cache_query_cb(void * data, const uchar * key, uint si
else else
{ {
struct promise *p = (struct promise *)user_arg; struct promise *p = (struct promise *)user_arg;
if (time(NULL) - kyr->update_time > ((struct key_keeper_promise_ctx *)promise_get_ctx(p))->ref_keeper->hash_expire_seconds)
{
return KEYRING_EXPIRE;
}
else
{
promise_success(p, data); promise_success(p, data);
return KEYRING_EXSITED; return KEYRING_EXSITED;
} }
} }
}
static struct keyring_private* generate_x509_keyring(X509* origin_cert, X509* ca, EVP_PKEY* cakey, int cert_expire_time) static struct keyring_private* generate_x509_keyring(X509* origin_cert, X509* ca, EVP_PKEY* cakey, int cert_expire_time)
@@ -389,11 +401,23 @@ static void certstore_rpc_on_succ(void* result, void* user)
if(!ctx->ref_keeper->no_cache) if(!ctx->ref_keeper->no_cache)
{ {
keyring_ref_inc(kyr); keyring_ref_inc(kyr);
kyr->update_time = time(NULL);
pthread_rwlock_wrlock(&(ctx->ref_keeper->rwlock));
if (MESA_htable_get_elem_num(ctx->ref_keeper->cert_cache) == ctx->ref_keeper->max_elem_num)
{
MESA_htable_del_oldest_manual(htable, NULL, ctx->ref_keeper->del_elem_num_once_when_full);
TFE_LOG_DEBUG(ctx->ref_keeper->logger, "Key keeper cache full: %d del: %d left: %d", ctx->ref_keeper->max_elem_num, ctx->ref_keeper->del_elem_num_once_when_full, MESA_htable_get_elem_num(ctx->ref_keeper->cert_cache));
}
int ret = MESA_htable_add(htable, key, key_len, (void*)kyr); int ret = MESA_htable_add(htable, key, key_len, (void*)kyr);
pthread_rwlock_unlock(&(ctx->ref_keeper->rwlock));
if(ret<0) if(ret<0)
{ {
key_keeper_free_keyring((struct keyring*)kyr); key_keeper_free_keyring((struct keyring*)kyr);
} }
else
{
TFE_LOG_DEBUG(ctx->ref_keeper->logger, "Key keeper cache add key: %s", ctx->key);
}
} }
ctx->ref_keeper->stat.new_issue++; ctx->ref_keeper->stat.new_issue++;
promise_success(p, (void*)kyr); promise_success(p, (void*)kyr);
@@ -425,17 +449,23 @@ static void key_keeper_free_serialized(void* data)
key_keeper_free_keyring(&(kyr->head)); key_keeper_free_keyring(&(kyr->head));
} }
static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int expire_seconds) static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int max_elem_num)
{ {
UNUSED int ret = 0; UNUSED int ret = 0;
unsigned max_num = slot_size * 4;
MESA_htable_handle htable = MESA_htable_born(); MESA_htable_handle htable = MESA_htable_born();
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_SCREEN_PRINT_CTRL, 0); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_SCREEN_PRINT_CTRL, 0);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 1); /*
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 16); * Use rwlock instead of htable mutex:
* MHO_THREAD_SAFE must be set to 0,
* MHO_MUTEX_NUM must be set to 1,
* MHO_EXPIRE_TIME must be set to 0,
* otherwise MESA_htable_del_oldest_manual() execution error
*/
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 0);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 1);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, slot_size); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, slot_size);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, max_num); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, max_elem_num);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, expire_seconds); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, 0);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_ELIMIMINATE_TYPE, ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_ELIMIMINATE_TYPE,
HASH_ELIMINATE_ALGO_FIFO); HASH_ELIMINATE_ALGO_FIFO);
ret = __wrapper_MESA_htable_set_opt_func(htable, MHO_CBFUN_DATA_FREE, ret = __wrapper_MESA_htable_set_opt_func(htable, MHO_CBFUN_DATA_FREE,
@@ -449,7 +479,11 @@ static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int
void key_keeper_destroy(struct key_keeper *keeper) void key_keeper_destroy(struct key_keeper *keeper)
{ {
if (!keeper->no_cache)
{
pthread_rwlock_destroy(&(keeper->rwlock));
MESA_htable_destroy(keeper->cert_cache, NULL); MESA_htable_destroy(keeper->cert_cache, NULL);
}
X509_free(keeper->trusted_ca_cert); X509_free(keeper->trusted_ca_cert);
EVP_PKEY_free(keeper->trusted_ca_key); EVP_PKEY_free(keeper->trusted_ca_key);
@@ -573,16 +607,22 @@ struct key_keeper* key_keeper_init(const char * profile, const char* section, vo
MESA_load_profile_uint_def(profile, section, "enable_health_check", &(keeper->enable_health_check), 1); MESA_load_profile_uint_def(profile, section, "enable_health_check", &(keeper->enable_health_check), 1);
MESA_load_profile_uint_def(profile, section, "cert_store_port", &(keeper->cert_store_port), 80); MESA_load_profile_uint_def(profile, section, "cert_store_port", &(keeper->cert_store_port), 80);
MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024*128);
MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5*60);
MESA_load_profile_uint_def(profile, section, "no_cache", &(keeper->no_cache), 0);
MESA_load_profile_int_def(profile, section, "cert_expire_time", &(keeper->cert_expire_time), 24); MESA_load_profile_int_def(profile, section, "cert_expire_time", &(keeper->cert_expire_time), 24);
keeper->cert_cache = create_hash_table(keeper->hash_slot_size, keeper->hash_expire_seconds); MESA_load_profile_uint_def(profile, section, "no_cache", &(keeper->no_cache), 0);
if (!keeper->no_cache)
{
MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5 * 60);
MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024 * 128);
if (keeper->cert_expire_time != -1) if (keeper->cert_expire_time != -1)
{ {
keeper->hash_expire_seconds = MIN(keeper->cert_expire_time * 1800, (int)(keeper->hash_expire_seconds)); keeper->hash_expire_seconds = MIN(keeper->cert_expire_time * 1800, (int)(keeper->hash_expire_seconds));
} }
keeper->max_elem_num = keeper->hash_slot_size * 4;
keeper->del_elem_num_once_when_full = (keeper->max_elem_num / 10) > 0 ? (keeper->max_elem_num / 10) : 1;
keeper->cert_cache = create_hash_table(keeper->hash_slot_size, keeper->max_elem_num);
pthread_rwlock_init(&(keeper->rwlock), NULL);
}
if(0==strcmp(keeper->untrusted_ca_path, keeper->trusted_ca_path)) if(0==strcmp(keeper->untrusted_ca_path, keeper->trusted_ca_path))
{ {
TFE_LOG_ERROR(logger, "Warnning: Trusted and Untrusted Root CA share the same path %s .", keeper->trusted_ca_path); TFE_LOG_ERROR(logger, "Warnning: Trusted and Untrusted Root CA share the same path %s .", keeper->trusted_ca_path);
@@ -620,8 +660,8 @@ struct key_keeper* key_keeper_init(const char * profile, const char* section, vo
} }
} }
TFE_LOG_INFO(logger, "MESA_load_profile, [%s]: mode:%s, no_cache:%u ,ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d, cert_expire_time:%d", TFE_LOG_INFO(logger, "Key keeper cache, mode:%s, no_cache:%u, ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d, cert_expire_time:%d, max_elem_num:%d, del_elem_num_once_when_full:%d",
section, tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds, keeper->cert_expire_time); tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds, keeper->cert_expire_time, keeper->max_elem_num, keeper->del_elem_num_once_when_full);
return keeper; return keeper;
@@ -696,12 +736,23 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c
keeper->stat.ask_times++; keeper->stat.ask_times++;
if(!keeper->no_cache) if(!keeper->no_cache)
{ {
char *tmp = tfe_strdup((const char *)ctx->key);
pthread_rwlock_rdlock(&(keeper->rwlock));
MESA_htable_search_cb(keeper->cert_cache, (const unsigned char*)(ctx->key), ctx->key_len, keyring_local_cache_query_cb, p, &cb_rtn); MESA_htable_search_cb(keeper->cert_cache, (const unsigned char*)(ctx->key), ctx->key_len, keyring_local_cache_query_cb, p, &cb_rtn);
pthread_rwlock_unlock(&(keeper->rwlock));
TFE_LOG_DEBUG(keeper->logger, "Key keeper cache search key: %s, found: %d (0:KEYRING_EXSITED, -1:KEYRING_NOT_EXSITED, -2:KEYRING_EXPIRE)", tmp, cb_rtn);
free(tmp);
if(cb_rtn == KEYRING_EXSITED) if(cb_rtn == KEYRING_EXSITED)
{ {
//printf("KEYRING_EXSITED\n"); //printf("KEYRING_EXSITED\n");
return; return;
} }
if (cb_rtn == KEYRING_EXPIRE)
{
pthread_rwlock_wrlock(&(keeper->rwlock));
MESA_htable_del(keeper->cert_cache, (const unsigned char *)(ctx->key), ctx->key_len, NULL);
pthread_rwlock_unlock(&(keeper->rwlock));
}
} }
switch(keeper->work_mode) switch(keeper->work_mode)
{ {
@@ -749,10 +800,22 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c
if(!keeper->no_cache) if(!keeper->no_cache)
{ {
keyring_ref_inc(kyr); keyring_ref_inc(kyr);
kyr->update_time = time(NULL);
pthread_rwlock_wrlock(&(keeper->rwlock));
if (MESA_htable_get_elem_num(keeper->cert_cache) == keeper->max_elem_num)
{
MESA_htable_del_oldest_manual(keeper->cert_cache, NULL, keeper->del_elem_num_once_when_full);
TFE_LOG_DEBUG(keeper->logger, "Key keeper cache full: %d del: %d left: %d", keeper->max_elem_num, keeper->del_elem_num_once_when_full, MESA_htable_get_elem_num(keeper->cert_cache));
}
int ret = MESA_htable_add(ctx->ref_keeper->cert_cache, ctx->key, ctx->key_len, (void *)kyr); int ret = MESA_htable_add(ctx->ref_keeper->cert_cache, ctx->key, ctx->key_len, (void *)kyr);
pthread_rwlock_unlock(&(keeper->rwlock));
if(ret < 0) if(ret < 0)
{ {
key_keeper_free_keyring((struct keyring*)kyr); key_keeper_free_keyring((struct keyring*)kyr);
}
else
{
TFE_LOG_DEBUG(keeper->logger, "Key keeper cache add key: %s", ctx->key);
} }
} }
promise_success(p, (void*)kyr); promise_success(p, (void*)kyr);
@@ -770,8 +833,12 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c
} }
void key_keeper_statistic(struct key_keeper *keeper, struct key_keeper_stat* result) void key_keeper_statistic(struct key_keeper *keeper, struct key_keeper_stat* result)
{ {
if (!keeper->no_cache)
{
// pthread_rwlock_rdlock(&(keeper->rwlock));
keeper->stat.cached_num=MESA_htable_get_elem_num(keeper->cert_cache); keeper->stat.cached_num=MESA_htable_get_elem_num(keeper->cert_cache);
// pthread_rwlock_unlock(&(keeper->rwlock));
}
*result = keeper->stat; *result = keeper->stat;
return; return;
} }

2
platform/src/ssl_stream.cpp
View File

@@ -352,7 +352,7 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT])); ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT]));
ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT])); ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT]));
} }
struct key_keeper_stat keeper_stat; struct key_keeper_stat keeper_stat = { 0 };
key_keeper_statistic(mgr->key_keeper, &keeper_stat); key_keeper_statistic(mgr->key_keeper, &keeper_stat);
mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times; mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times;
mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue; mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue;