diff --git a/platform/src/key_keeper.cpp b/platform/src/key_keeper.cpp index fa932df..a48854b 100644 --- a/platform/src/key_keeper.cpp +++ b/platform/src/key_keeper.cpp @@ -23,6 +23,7 @@ #define HTABLE_MAX_KEY_LEN 256 #define KEYRING_EXSITED 0 #define KEYRING_NOT_EXSITED -1 +#define KEYRING_EXPIRE -2 enum key_keeper_mode{ KK_MODE_CERT_STORE = 0, @@ -37,7 +38,9 @@ struct key_keeper char cert_store_host[TFE_SYMBOL_MAX]; unsigned int cert_store_port; unsigned int hash_slot_size; - unsigned int hash_expire_seconds; + unsigned int hash_expire_seconds; + unsigned int del_elem_num_once_when_full; + unsigned int max_elem_num; MESA_htable_handle cert_cache; void* logger; X509* trusted_ca_cert; @@ -51,6 +54,7 @@ struct key_keeper int cert_expire_time; unsigned int enable_health_check; pthread_t thread; + pthread_rwlock_t rwlock; }; @@ -60,6 +64,7 @@ struct keyring_private struct keyring head; pthread_mutex_t mutex; size_t references; + time_t update_time; }; struct key_keeper_promise_ctx @@ -335,9 +340,16 @@ static long keyring_local_cache_query_cb(void * data, const uchar * key, uint si } else { - struct promise* p = (struct promise*)user_arg; - promise_success(p, data); - return KEYRING_EXSITED; + struct promise *p = (struct promise *)user_arg; + if (time(NULL) - kyr->update_time > ((struct key_keeper_promise_ctx *)promise_get_ctx(p))->ref_keeper->hash_expire_seconds) + { + return KEYRING_EXPIRE; + } + else + { + promise_success(p, data); + return KEYRING_EXSITED; + } } } @@ -389,12 +401,24 @@ static void certstore_rpc_on_succ(void* result, void* user) if(!ctx->ref_keeper->no_cache) { keyring_ref_inc(kyr); - int ret = MESA_htable_add(htable, key, key_len, (void*)kyr); - if(ret<0) + kyr->update_time = time(NULL); + pthread_rwlock_wrlock(&(ctx->ref_keeper->rwlock)); + if (MESA_htable_get_elem_num(ctx->ref_keeper->cert_cache) == ctx->ref_keeper->max_elem_num) + { + MESA_htable_del_oldest_manual(htable, NULL, ctx->ref_keeper->del_elem_num_once_when_full); + TFE_LOG_DEBUG(ctx->ref_keeper->logger, "Key keeper cache full: %d del: %d left: %d", ctx->ref_keeper->max_elem_num, ctx->ref_keeper->del_elem_num_once_when_full, MESA_htable_get_elem_num(ctx->ref_keeper->cert_cache)); + } + int ret = MESA_htable_add(htable, key, key_len, (void*)kyr); + pthread_rwlock_unlock(&(ctx->ref_keeper->rwlock)); + if(ret<0) { key_keeper_free_keyring((struct keyring*)kyr); } - } + else + { + TFE_LOG_DEBUG(ctx->ref_keeper->logger, "Key keeper cache add key: %s", ctx->key); + } + } ctx->ref_keeper->stat.new_issue++; promise_success(p, (void*)kyr); key_keeper_free_keyring((struct keyring*)kyr); @@ -425,17 +449,23 @@ static void key_keeper_free_serialized(void* data) key_keeper_free_keyring(&(kyr->head)); } -static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int expire_seconds) +static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int max_elem_num) { UNUSED int ret = 0; - unsigned max_num = slot_size * 4; MESA_htable_handle htable = MESA_htable_born(); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_SCREEN_PRINT_CTRL, 0); - ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 1); - ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 16); - ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, slot_size); - ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, max_num); - ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, expire_seconds); + /* + * Use rwlock instead of htable mutex: + * MHO_THREAD_SAFE must be set to 0, + * MHO_MUTEX_NUM must be set to 1, + * MHO_EXPIRE_TIME must be set to 0, + * otherwise MESA_htable_del_oldest_manual() execution error + */ + ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 0); + ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 1); + ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, slot_size); + ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, max_elem_num); + ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, 0); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_ELIMIMINATE_TYPE, HASH_ELIMINATE_ALGO_FIFO); ret = __wrapper_MESA_htable_set_opt_func(htable, MHO_CBFUN_DATA_FREE, @@ -449,7 +479,11 @@ static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int void key_keeper_destroy(struct key_keeper *keeper) { - MESA_htable_destroy(keeper->cert_cache, NULL); + if (!keeper->no_cache) + { + pthread_rwlock_destroy(&(keeper->rwlock)); + MESA_htable_destroy(keeper->cert_cache, NULL); + } X509_free(keeper->trusted_ca_cert); EVP_PKEY_free(keeper->trusted_ca_key); @@ -573,16 +607,22 @@ struct key_keeper* key_keeper_init(const char * profile, const char* section, vo MESA_load_profile_uint_def(profile, section, "enable_health_check", &(keeper->enable_health_check), 1); MESA_load_profile_uint_def(profile, section, "cert_store_port", &(keeper->cert_store_port), 80); - MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024*128); - MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5*60); - MESA_load_profile_uint_def(profile, section, "no_cache", &(keeper->no_cache), 0); + MESA_load_profile_int_def(profile, section, "cert_expire_time", &(keeper->cert_expire_time), 24); + MESA_load_profile_uint_def(profile, section, "no_cache", &(keeper->no_cache), 0); + if (!keeper->no_cache) + { + MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5 * 60); + MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024 * 128); + if (keeper->cert_expire_time != -1) + { + keeper->hash_expire_seconds = MIN(keeper->cert_expire_time * 1800, (int)(keeper->hash_expire_seconds)); + } - MESA_load_profile_int_def(profile, section, "cert_expire_time", &(keeper->cert_expire_time), 24); - keeper->cert_cache = create_hash_table(keeper->hash_slot_size, keeper->hash_expire_seconds); - if (keeper->cert_expire_time != -1) - { - keeper->hash_expire_seconds = MIN(keeper->cert_expire_time * 1800, (int)(keeper->hash_expire_seconds)); - } + keeper->max_elem_num = keeper->hash_slot_size * 4; + keeper->del_elem_num_once_when_full = (keeper->max_elem_num / 10) > 0 ? (keeper->max_elem_num / 10) : 1; + keeper->cert_cache = create_hash_table(keeper->hash_slot_size, keeper->max_elem_num); + pthread_rwlock_init(&(keeper->rwlock), NULL); + } if(0==strcmp(keeper->untrusted_ca_path, keeper->trusted_ca_path)) { TFE_LOG_ERROR(logger, "Warnning: Trusted and Untrusted Root CA share the same path %s .", keeper->trusted_ca_path); @@ -619,11 +659,11 @@ struct key_keeper* key_keeper_init(const char * profile, const char* section, vo goto error_out; } } - - TFE_LOG_INFO(logger, "MESA_load_profile, [%s]: mode:%s, no_cache:%u ,ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d, cert_expire_time:%d", - section, tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds, keeper->cert_expire_time); - return keeper; + TFE_LOG_INFO(logger, "Key keeper cache, mode:%s, no_cache:%u, ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d, cert_expire_time:%d, max_elem_num:%d, del_elem_num_once_when_full:%d", + tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds, keeper->cert_expire_time, keeper->max_elem_num, keeper->del_elem_num_once_when_full); + + return keeper; error_out: key_keeper_destroy(keeper); @@ -696,13 +736,24 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c keeper->stat.ask_times++; if(!keeper->no_cache) { - MESA_htable_search_cb(keeper->cert_cache, (const unsigned char*)(ctx->key), ctx->key_len, keyring_local_cache_query_cb, p, &cb_rtn); - if(cb_rtn == KEYRING_EXSITED) + char *tmp = tfe_strdup((const char *)ctx->key); + pthread_rwlock_rdlock(&(keeper->rwlock)); + MESA_htable_search_cb(keeper->cert_cache, (const unsigned char*)(ctx->key), ctx->key_len, keyring_local_cache_query_cb, p, &cb_rtn); + pthread_rwlock_unlock(&(keeper->rwlock)); + TFE_LOG_DEBUG(keeper->logger, "Key keeper cache search key: %s, found: %d (0:KEYRING_EXSITED, -1:KEYRING_NOT_EXSITED, -2:KEYRING_EXPIRE)", tmp, cb_rtn); + free(tmp); + if(cb_rtn == KEYRING_EXSITED) { //printf("KEYRING_EXSITED\n"); return; } - } + if (cb_rtn == KEYRING_EXPIRE) + { + pthread_rwlock_wrlock(&(keeper->rwlock)); + MESA_htable_del(keeper->cert_cache, (const unsigned char *)(ctx->key), ctx->key_len, NULL); + pthread_rwlock_unlock(&(keeper->rwlock)); + } + } switch(keeper->work_mode) { case KK_MODE_CERT_STORE: @@ -749,11 +800,23 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c if(!keeper->no_cache) { keyring_ref_inc(kyr); - int ret = MESA_htable_add(ctx->ref_keeper->cert_cache, ctx->key, ctx->key_len, (void*)kyr); - if(ret < 0) + kyr->update_time = time(NULL); + pthread_rwlock_wrlock(&(keeper->rwlock)); + if (MESA_htable_get_elem_num(keeper->cert_cache) == keeper->max_elem_num) + { + MESA_htable_del_oldest_manual(keeper->cert_cache, NULL, keeper->del_elem_num_once_when_full); + TFE_LOG_DEBUG(keeper->logger, "Key keeper cache full: %d del: %d left: %d", keeper->max_elem_num, keeper->del_elem_num_once_when_full, MESA_htable_get_elem_num(keeper->cert_cache)); + } + int ret = MESA_htable_add(ctx->ref_keeper->cert_cache, ctx->key, ctx->key_len, (void *)kyr); + pthread_rwlock_unlock(&(keeper->rwlock)); + if(ret < 0) { key_keeper_free_keyring((struct keyring*)kyr); } + else + { + TFE_LOG_DEBUG(keeper->logger, "Key keeper cache add key: %s", ctx->key); + } } promise_success(p, (void*)kyr); keeper->stat.new_issue++; @@ -770,8 +833,12 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c } void key_keeper_statistic(struct key_keeper *keeper, struct key_keeper_stat* result) { - keeper->stat.cached_num=MESA_htable_get_elem_num(keeper->cert_cache); - *result=keeper->stat; - return; -} - + if (!keeper->no_cache) + { + // pthread_rwlock_rdlock(&(keeper->rwlock)); + keeper->stat.cached_num=MESA_htable_get_elem_num(keeper->cert_cache); + // pthread_rwlock_unlock(&(keeper->rwlock)); + } + *result = keeper->stat; + return; +} \ No newline at end of file diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index b2db7b6..1b151a4 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -352,7 +352,7 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT])); ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT])); } - struct key_keeper_stat keeper_stat; + struct key_keeper_stat keeper_stat = { 0 }; key_keeper_statistic(mgr->key_keeper, &keeper_stat); mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times; mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue;