#include "key_keeper.h" #include "MESA_htable_aux.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #define HTABLE_MAX_KEY_LEN 256 #define KEYRING_EXSITED 0 #define KEYRING_NOT_EXSITED -1 enum key_keeper_mode{ KK_MODE_CERT_STORE = 0, KK_MODE_LOCAL }; struct key_keeper { enum key_keeper_mode work_mode; char trusted_ca_path[TFE_PATH_MAX]; char untrusted_ca_path[TFE_PATH_MAX]; char cert_store_host[TFE_SYMBOL_MAX]; unsigned int cert_store_port; unsigned int hash_slot_size; unsigned int hash_expire_seconds; MESA_htable_handle cert_cache; void* logger; X509* trusted_ca_cert; EVP_PKEY* trusted_ca_key; X509* untrusted_ca_cert; EVP_PKEY* untrusted_ca_key; unsigned int no_cache; struct key_keeper_stat stat; }; struct keyring_private { struct keyring head; pthread_mutex_t mutex; size_t references; }; struct key_keeper_promise_ctx { void* logger; struct key_keeper* ref_keeper; uchar* key; struct future* f_certstore_rpc; unsigned int key_len; }; static void key_keeper_promise_free_ctx(void* ctx) { struct key_keeper_promise_ctx* _ctx = (struct key_keeper_promise_ctx*)ctx; free(_ctx->key); _ctx->key = NULL; free(_ctx); } static struct keyring_private* keyring_new(X509 *cert, EVP_PKEY *key, STACK_OF(X509) *chain) { struct keyring_private *kyr=ALLOC(struct keyring_private, 1); pthread_mutex_init(&(kyr->mutex), NULL); kyr->head.cert = cert; kyr->head.key = key; kyr->head.chain = chain; kyr->references = 1; return kyr; } /* static struct keyring* keyring_new3(EVP_PKEY *key, X509 *cert, STACK_OF(X509) *chain) { struct keyring_private* kyr=NULL; kyr=keyring_new(); kyr->references = 1; (kyr->head).key = key; (kyr->head).cert = cert; (kyr->head).chain = chain; if(key) { ssl_key_refcount_inc(key); } if(cert) { ssl_x509_refcount_inc(cert); } if(chain){ int i = 0; for (i = 0; i < sk_X509_num(chain); i++) { ssl_x509_refcount_inc(sk_X509_value(chain, i)); } } return &(kyr->head); } */ // Increment reference count. static void keyring_ref_inc(struct keyring_private* kyr) { pthread_mutex_lock(&(kyr->mutex)); kyr->references++; pthread_mutex_unlock(&(kyr->mutex)); } /* * Free keyring including internal objects. */ void key_keeper_free_keyring(struct keyring *kyr) { struct keyring_private* _kyr = (struct keyring_private*)kyr; pthread_mutex_lock(&_kyr->mutex); _kyr->references--; if (_kyr->references>0) { pthread_mutex_unlock(&_kyr->mutex); return; } pthread_mutex_unlock(&_kyr->mutex); pthread_mutex_destroy(&_kyr->mutex); if (_kyr->head.key) { EVP_PKEY_free((_kyr->head).key); _kyr->head.key=NULL; } if (_kyr->head.cert) { X509_free(_kyr->head.cert); _kyr->head.cert=NULL; } if (_kyr->head.chain) { sk_X509_pop_free((_kyr->head).chain, X509_free); _kyr->head.chain=NULL; } free(_kyr); } static X509* transform_cert_to_x509(const char* str) { //printf("cert: %s", str); BIO *bio = NULL; X509 *cert = NULL; bio = BIO_new(BIO_s_mem()); if(bio == NULL) { return NULL; } int len = BIO_write(bio, (const void*)str, strlen(str)); if (len <= 0 ) { BIO_free_all(bio); return NULL; } cert = PEM_read_bio_X509(bio, NULL, NULL, NULL); BIO_free_all(bio); return cert; } static char* transform_cert_to_pem(X509* cert) { if (NULL == cert) { return NULL; } BIO* bio = NULL; bio = BIO_new(BIO_s_mem()); if (NULL == bio) { return NULL; } if (0 == PEM_write_bio_X509(bio, cert)) { BIO_free(bio); return NULL; } char *p = NULL; int len = BIO_get_mem_data(bio, &p); char *pem = (char*)malloc(len + 1); memset(pem, 0, len + 1); BIO_read(bio, pem, len); BIO_free(bio); return pem; } static EVP_PKEY* transform_key_to_EVP(const char* str) { //printf("private key: %s", str); BIO *mem; mem = BIO_new_mem_buf(str, -1); if(mem == NULL) { return NULL; } EVP_PKEY* key = PEM_read_bio_PrivateKey(mem, NULL, NULL, 0); BIO_free(mem); return key; } static struct keyring_private* get_keyring_from_response(const char* data) { X509* cert = NULL; X509* chain_cert = NULL; struct keyring_private* _kyr=NULL; EVP_PKEY* key = NULL; STACK_OF(X509)* chain = NULL; cJSON* data_json = NULL; cJSON* chain_cert_json = NULL; cJSON* cert_json = NULL; cJSON* key_json = NULL; cJSON* chain_json = NULL; assert(data != NULL); data_json = cJSON_Parse(data); if(unlikely(data_json == NULL)) { TFE_LOG_ERROR(g_default_logger, "Illegal JSON format: %s", data); goto error_out; } cert_json = cJSON_GetObjectItemCaseSensitive(data_json, "CERTIFICATE"); key_json = cJSON_GetObjectItemCaseSensitive(data_json, "PRIVATE_KEY"); chain_json = cJSON_GetObjectItemCaseSensitive(data_json, "CERTIFICATE_CHAIN"); if(unlikely(cert_json == NULL)) { TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No CERTIFICATE section: %s", data); goto error_out; } if(unlikely(key_json == NULL)) { TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No PRIVATE_KEY section: %s", data); goto error_out; } if(unlikely(chain_json == NULL)) { TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No CERTIFICATE_CHAIN section: %s", data); goto error_out; } if(unlikely(cert_json->valuestring == NULL)) { TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No CERTIFICATE value: %s", data); goto error_out; } if(unlikely(key_json->valuestring == NULL)) { TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No PRIVATE_KEY value: %s", data); goto error_out; } cert = transform_cert_to_x509(cert_json->valuestring); if(unlikely(cert == NULL)) { TFE_LOG_ERROR(g_default_logger, "Transform certificate to X509 failed: %s", cert_json->valuestring); goto error_out; } key = transform_key_to_EVP(key_json->valuestring); if(unlikely(key == NULL)) { TFE_LOG_ERROR(g_default_logger, "Transform PRIVATE KEY to EVP failed: %s", key_json->valuestring); goto error_out; } chain = sk_X509_new_null(); cJSON_ArrayForEach(chain_cert_json, chain_json) { if(unlikely(chain_cert_json->valuestring == NULL)) { TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, empty CERTIFICATE_CHAIN value."); goto error_out; } chain_cert = transform_cert_to_x509(chain_cert_json->valuestring); if(unlikely(chain_cert == NULL)) { TFE_LOG_ERROR(g_default_logger, "Transform certificate chain entry to X509 failed: %s", chain_cert_json->valuestring); goto error_out; } sk_X509_push(chain, chain_cert); } _kyr= keyring_new(cert, key, chain); cJSON_Delete(data_json); return _kyr; error_out: if(data_json!=NULL) cJSON_Delete(data_json); if(cert) X509_free(cert); if(key) EVP_PKEY_free(key); if(chain) sk_X509_pop_free(chain, X509_free); return NULL; } static long keyring_local_cache_query_cb(void * data, const uchar * key, uint size, void * user_arg) { struct keyring_private* kyr=(struct keyring_private*)data; if(kyr == NULL) { return KEYRING_NOT_EXSITED; } else { struct promise* p = (struct promise*)user_arg; promise_success(p, data); return KEYRING_EXSITED; } } static struct keyring_private* generate_x509_keyring(X509* origin_cert, X509* ca, EVP_PKEY* cakey) { //TODO: could be optimized to save cpu. EVP_PKEY* forge_key = ssl_key_genrsa(2048); X509* forge_cert = ssl_x509_forge(ca, cakey, origin_cert, forge_key, NULL, NULL); STACK_OF(X509)* chain = sk_X509_new_null(); sk_X509_push(chain, ca); ssl_x509_refcount_inc(ca); ssl_x509_refcount_inc(forge_cert); struct keyring_private* _kyr= keyring_new(forge_cert, forge_key, chain); return _kyr; } static void certstore_rpc_on_succ(void* result, void* user) { struct promise * p = (struct promise *) user; struct key_keeper_promise_ctx* ctx = (struct key_keeper_promise_ctx*)promise_get_ctx(p); // TFE_LOG_INFO(ctx->logger, "certstore rpc success"); future_destroy(ctx->f_certstore_rpc); MESA_htable_handle htable= ctx->ref_keeper->cert_cache; const uchar* key = ctx->key; unsigned int key_len = ctx->key_len; struct tfe_rpc_response_result* response = tfe_rpc_release(result); int status_code = response->status_code; const char* status_msg = response->status_msg; if(status_code == HTTP_OK) { /* Copy a buffer ending with zero */ char * data_as_string = (char *)malloc(response->len + 1); memcpy(data_as_string, response->data, response->len); data_as_string[response->len] = '\0'; struct keyring_private* kyr= get_keyring_from_response(data_as_string); FREE(&data_as_string); if(kyr == NULL) { promise_failed(p, FUTURE_ERROR_EXCEPTION, "get_keyring_from_response failed"); return; } if(!ctx->ref_keeper->no_cache) { keyring_ref_inc(kyr); int ret = MESA_htable_add(htable, key, key_len, (void*)kyr); if(ret<0) { key_keeper_free_keyring((struct keyring*)kyr); } } ctx->ref_keeper->stat.new_issue++; promise_success(p, (void*)kyr); key_keeper_free_keyring((struct keyring*)kyr); } else { promise_failed(p, FUTURE_ERROR_EXCEPTION, status_msg); } } static void certstore_rpc_on_fail(enum e_future_error err, const char * what, void * user) { struct promise * p = (struct promise *) user; struct key_keeper_promise_ctx* ctx= (struct key_keeper_promise_ctx*)promise_get_ctx(p); TFE_LOG_ERROR(ctx->logger, "certstore rpc failed, what is %s", what); future_destroy(ctx->f_certstore_rpc); promise_failed(p, err, what); //promise_dettach_ctx(p); //ctx_destroy_cb((void*)ctx); } static void key_keeper_free_serialized(void* data) { // printf("call key_keeper_free_serialized\n"); struct keyring_private* kyr = (struct keyring_private*)data; key_keeper_free_keyring(&(kyr->head)); } static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int expire_seconds) { UNUSED int ret = 0; unsigned max_num = slot_size * 4; MESA_htable_handle htable = MESA_htable_born(); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_SCREEN_PRINT_CTRL, 0); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 1); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 16); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, slot_size); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, max_num); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, expire_seconds); ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_ELIMIMINATE_TYPE, HASH_ELIMINATE_ALGO_FIFO); ret = __wrapper_MESA_htable_set_opt_func(htable, MHO_CBFUN_DATA_FREE, (void *)key_keeper_free_serialized, sizeof(&key_keeper_free_serialized)); //ret = __wrapper_MESA_htable_set_opt(htable, MHO_CBFUN_DATA_EXPIRE_NOTIFY, // (void *)key_keeper_verify_cb); ret = MESA_htable_mature(htable); assert(ret == 0); return htable; } void key_keeper_destroy(struct key_keeper *keeper) { MESA_htable_destroy(keeper->cert_cache, NULL); X509_free(keeper->trusted_ca_cert); EVP_PKEY_free(keeper->trusted_ca_key); X509_free(keeper->untrusted_ca_cert); EVP_PKEY_free(keeper->untrusted_ca_key); free(keeper); keeper = NULL; return; } struct key_keeper* key_keeper_init(const char * profile, const char* section, void* logger) { struct key_keeper* keeper = ALLOC(struct key_keeper, 1); keeper->logger = logger; char tmp[TFE_STRING_MAX]={0}; MESA_load_profile_string_def(profile, section, "mode", tmp, sizeof(tmp), "debug"); if(strcasecmp(tmp, "debug") == 0) { keeper->work_mode = KK_MODE_LOCAL; } else { keeper->work_mode = KK_MODE_CERT_STORE; } MESA_load_profile_string_def(profile, section, "ca_path", keeper->trusted_ca_path, sizeof(keeper->trusted_ca_path), "./resource/tfe/mesalab-ca.pem"); MESA_load_profile_string_def(profile, section, "untrusted_ca_path", keeper->untrusted_ca_path, sizeof(keeper->untrusted_ca_path), "./resource/tfe/mesalab-ca-untrust.pem"); MESA_load_profile_string_def(profile, section, "cert_store_host", keeper->cert_store_host, sizeof(keeper->cert_store_host), ""); MESA_load_profile_uint_def(profile, section, "cert_store_port", &(keeper->cert_store_port), 80); MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024*128); MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5*60); MESA_load_profile_uint_def(profile, section, "no_cache", &(keeper->no_cache), 0); keeper->cert_cache = create_hash_table(keeper->hash_slot_size, keeper->hash_expire_seconds); if(0==strcmp(keeper->untrusted_ca_path, keeper->trusted_ca_path)) { TFE_LOG_ERROR(logger, "Warnning: Trusted and Untrusted Root CA share the same path %s .", keeper->trusted_ca_path); } if(keeper->work_mode==KK_MODE_LOCAL) { keeper->trusted_ca_cert=ssl_x509_load(keeper->trusted_ca_path); keeper->trusted_ca_key=ssl_key_load(keeper->trusted_ca_path); if(keeper->trusted_ca_cert==NULL||keeper->trusted_ca_key==NULL) { TFE_LOG_ERROR(logger, "Load Trusted Root CA %s failed.", keeper->trusted_ca_path); goto error_out; } keeper->untrusted_ca_cert=ssl_x509_load(keeper->untrusted_ca_path); keeper->untrusted_ca_key=ssl_key_load(keeper->untrusted_ca_path); if(keeper->untrusted_ca_cert==NULL||keeper->trusted_ca_key==NULL) { TFE_LOG_ERROR(logger, "Load Untrusted Root CA %s failed.", keeper->untrusted_ca_path); goto error_out; } } TFE_LOG_INFO(logger, "MESA_load_profile, [%s]: mode:%s, no_cache:%u ,ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d", section, tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds); return keeper; error_out: key_keeper_destroy(keeper); return NULL; } struct keyring* key_keeper_release_keyring(future_result_t* result) { struct keyring_private* kyr=(struct keyring_private*)result; keyring_ref_inc(kyr); return &(kyr->head); } static uchar* get_key_by_cert(X509* cert, int keyring_id, unsigned int* len, int is_cert_valid) { if(cert == NULL) { return NULL; } char* cert_fingerprint = NULL; cert_fingerprint = ssl_x509_fingerprint(cert, 0); if(cert_fingerprint == NULL) { return NULL; } char* key = ALLOC(char, HTABLE_MAX_KEY_LEN); memset(key, 0, HTABLE_MAX_KEY_LEN); snprintf(key, HTABLE_MAX_KEY_LEN, "%d:%d:", keyring_id, is_cert_valid); strncat(key, cert_fingerprint, HTABLE_MAX_KEY_LEN); *len = strnlen(key, HTABLE_MAX_KEY_LEN); free(cert_fingerprint); return (uchar*)key; } char* url_escape(char* url) { if(url == NULL) { return NULL; } CURL *curl = curl_easy_init(); char* _url = NULL; if(curl) { _url = curl_easy_escape(curl, url, strlen(url)); } curl_easy_cleanup(curl); return _url; } void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const char* sni, int keyring_id, X509 * origin_cert, int is_cert_valid, struct event_base * evbase, struct evdns_base* dnsbase) { struct promise* p = future_to_promise(f); unsigned int len = 0; uchar* key = get_key_by_cert(origin_cert, keyring_id, &len, is_cert_valid); if(key == NULL) { promise_failed(p, FUTURE_ERROR_EXCEPTION, "get hash key by_cert failed"); return; } struct key_keeper_promise_ctx* ctx = ALLOC(struct key_keeper_promise_ctx, 1); ctx->logger = keeper->logger; ctx->ref_keeper = keeper; ctx->key = key; ctx->key_len = len; promise_set_ctx(p, (void*)ctx, key_keeper_promise_free_ctx); long int cb_rtn = 0; keeper->stat.ask_times++; if(!keeper->no_cache) { MESA_htable_search_cb(keeper->cert_cache, (const unsigned char*)(ctx->key), ctx->key_len, keyring_local_cache_query_cb, p, &cb_rtn); if(cb_rtn == KEYRING_EXSITED) { //printf("KEYRING_EXSITED\n"); return; } } switch(keeper->work_mode) { case KK_MODE_CERT_STORE: { char* origin_cert_pem = transform_cert_to_pem(origin_cert); if(origin_cert_pem == NULL) { promise_failed(p, FUTURE_ERROR_EXCEPTION, "transform origin_cert to pem failed"); return; } char* escaped_origin_cert_pem = url_escape(origin_cert_pem); free(origin_cert_pem); if(escaped_origin_cert_pem == NULL) { promise_failed(p, FUTURE_ERROR_EXCEPTION, "url escape failed"); break; } struct future* f_certstore_rpc = future_create("crt_store", certstore_rpc_on_succ, certstore_rpc_on_fail, p); ctx->f_certstore_rpc = f_certstore_rpc; char *url = NULL; //keyring_id = 1; if(sni == NULL || sni[0] == '\0') { asprintf(&url, "http://%s:%d/ca?keyring_id=%d&is_valid=%d&origin_cert=%s", keeper->cert_store_host, keeper->cert_store_port, keyring_id, is_cert_valid, escaped_origin_cert_pem); } else { asprintf(&url, "http://%s:%d/ca?keyring_id=%d&sni=%s&is_valid=%d&origin_cert=%s", keeper->cert_store_host, keeper->cert_store_port, keyring_id, sni, is_cert_valid, escaped_origin_cert_pem); } TFE_LOG_DEBUG(keeper->logger, "CertStore query: %.100s", url); curl_free(escaped_origin_cert_pem); tfe_rpc_async_ask(f_certstore_rpc, url, GET, DONE_CB, NULL, 0, evbase, dnsbase); free(url); break; } case KK_MODE_LOCAL: { struct keyring_private* kyr=NULL; if(is_cert_valid == 1) { kyr=generate_x509_keyring(origin_cert, keeper->trusted_ca_cert, keeper->trusted_ca_key); } else { kyr=generate_x509_keyring(origin_cert, keeper->untrusted_ca_cert, keeper->untrusted_ca_key); } if(kyr) { if(!keeper->no_cache) { keyring_ref_inc(kyr); int ret = MESA_htable_add(ctx->ref_keeper->cert_cache, ctx->key, ctx->key_len, (void*)kyr); if(ret < 0) { key_keeper_free_keyring((struct keyring*)kyr); } } promise_success(p, (void*)kyr); keeper->stat.new_issue++; key_keeper_free_keyring((struct keyring*)kyr); } else { promise_failed(p, FUTURE_ERROR_EXCEPTION, "generate X509 cert failed"); } break; } } return; } void key_keeper_statistic(struct key_keeper *keeper, struct key_keeper_stat* result) { keeper->stat.cached_num=MESA_htable_get_elem_num(keeper->cert_cache); *result=keeper->stat; return; }