tango-tfe/platform/src/key_keeper.cpp

#include "key_keeper.h"
#include "MESA_htable_aux.h"

#include <pthread.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <unistd.h>
#include <sys/prctl.h>
#include <arpa/inet.h>

#include <MESA/MESA_prof_load.h>
#include <MESA/MESA_htable.h>

#include <ssl_utils.h>
#include <tfe_rpc.h>
#include <tfe_utils.h>
#include <event2/http.h>
#include <cjson/cJSON.h>
#include <curl/curl.h>

#define HTABLE_MAX_KEY_LEN 256
#define KEYRING_EXSITED 0
#define KEYRING_NOT_EXSITED -1
#define KEYRING_EXPIRE -2

enum key_keeper_mode{
	KK_MODE_CERT_STORE = 0,
	KK_MODE_LOCAL
};

struct key_keeper
{
	enum key_keeper_mode work_mode;
	char trusted_ca_path[TFE_PATH_MAX];
	char untrusted_ca_path[TFE_PATH_MAX];
	char cert_store_host[TFE_SYMBOL_MAX];
	unsigned int cert_store_port;
	unsigned int hash_slot_size;
    unsigned int hash_expire_seconds;
    unsigned int del_elem_num_once_when_full;
    unsigned int max_elem_num;
    MESA_htable_handle cert_cache;
    void* logger;
	X509* trusted_ca_cert;
	EVP_PKEY* trusted_ca_key;

	X509* untrusted_ca_cert;
	EVP_PKEY* untrusted_ca_key;
	unsigned int no_cache;
	struct key_keeper_stat stat;

	int cert_expire_time;
	unsigned int enable_health_check;
	pthread_t thread;
    pthread_rwlock_t rwlock;
};



struct keyring_private
{
	struct keyring head;
	pthread_mutex_t mutex;
	size_t references;
    time_t update_time;
};

struct key_keeper_promise_ctx
{
	void* logger;
	struct key_keeper* ref_keeper;
	uchar* key;
	struct future* f_certstore_rpc;
	unsigned int key_len;
};

long long certstore_is_unavailable = 0;

static void key_keeper_promise_free_ctx(void* ctx)
{
	struct key_keeper_promise_ctx* _ctx = (struct key_keeper_promise_ctx*)ctx;
	free(_ctx->key);
	_ctx->key = NULL;
	free(_ctx);
}

static struct keyring_private* keyring_new(X509 *cert, EVP_PKEY *key, STACK_OF(X509) *chain)
{
	struct keyring_private *kyr=ALLOC(struct keyring_private, 1);
	pthread_mutex_init(&(kyr->mutex), NULL);
	kyr->head.cert = cert;
	kyr->head.key = key;
	kyr->head.chain = chain;
	kyr->references = 1;
	return kyr;
}

/*
static struct keyring* keyring_new3(EVP_PKEY *key, X509 *cert, STACK_OF(X509) *chain)
{
	struct keyring_private* kyr=NULL;
	kyr=keyring_new();

	kyr->references = 1;
	(kyr->head).key = key;
	(kyr->head).cert = cert;
	(kyr->head).chain = chain;
	if(key)
	{
		ssl_key_refcount_inc(key);
	}
	if(cert)
	{
		ssl_x509_refcount_inc(cert);
	}
	if(chain){
        int i = 0;
		for (i = 0; i < sk_X509_num(chain); i++)
		{
			ssl_x509_refcount_inc(sk_X509_value(chain, i));
		}
	}
	return &(kyr->head);
}
*/

// Increment reference count.
static void keyring_ref_inc(struct keyring_private* kyr)
{
	pthread_mutex_lock(&(kyr->mutex));
	kyr->references++;
	pthread_mutex_unlock(&(kyr->mutex));
}

/*
 * Free keyring including internal objects.
 */
void key_keeper_free_keyring(struct keyring *kyr)
{
	struct keyring_private* _kyr = (struct keyring_private*)kyr;
	pthread_mutex_lock(&_kyr->mutex);
	_kyr->references--;
	if (_kyr->references>0)
	{
		pthread_mutex_unlock(&_kyr->mutex);
		return;
	}
	pthread_mutex_unlock(&_kyr->mutex);
	pthread_mutex_destroy(&_kyr->mutex);
	if (_kyr->head.key)
	{
		EVP_PKEY_free((_kyr->head).key);
		_kyr->head.key=NULL;
	}
	if (_kyr->head.cert)
	{
		X509_free(_kyr->head.cert);
		_kyr->head.cert=NULL;
	}
	if (_kyr->head.chain)
	{
		sk_X509_pop_free((_kyr->head).chain, X509_free);
		_kyr->head.chain=NULL;
	}
	free(_kyr);
}


static X509* transform_cert_to_x509(const char* str)
{
	//printf("cert: %s", str);
	BIO *bio = NULL;
	X509 *cert = NULL;
	bio = BIO_new(BIO_s_mem());
	if(bio == NULL)
	{
		return NULL;
	}
	int len = BIO_write(bio, (const void*)str, strlen(str));
	if (len <= 0 )
	{
		BIO_free_all(bio);
		return NULL;
	}
	cert = PEM_read_bio_X509(bio, NULL, NULL, NULL);
	BIO_free_all(bio);
	return cert;
}

static char* transform_cert_to_pem(X509* cert)
{
    if (NULL == cert)
	{
        return NULL;
    }
	BIO* bio = NULL;
    bio = BIO_new(BIO_s_mem());
    if (NULL == bio)
	{
        return NULL;
    }
    if (0 == PEM_write_bio_X509(bio, cert))
	{
        BIO_free(bio);
        return NULL;
    }
	char *p = NULL;
	int len = BIO_get_mem_data(bio, &p);
    char *pem = (char*)malloc(len + 1);
    memset(pem, 0, len + 1);
    BIO_read(bio, pem, len);
    BIO_free(bio);
    return pem;
}

static EVP_PKEY* transform_key_to_EVP(const char* str)
{
	//printf("private key: %s", str);
	BIO *mem;
	mem = BIO_new_mem_buf(str, -1);
	if(mem == NULL)
	{
		return NULL;
	}
	EVP_PKEY* key = PEM_read_bio_PrivateKey(mem, NULL, NULL, 0);
	BIO_free(mem);
	return key;
}

static struct keyring_private* get_keyring_from_response(const char* data)
{
	X509* cert = NULL;
	X509* chain_cert = NULL;
	struct keyring_private* _kyr=NULL;

	EVP_PKEY* key = NULL;
	STACK_OF(X509)* chain = NULL;
	cJSON* data_json = NULL;
	cJSON* chain_cert_json = NULL;
	cJSON* cert_json = NULL;
	cJSON* key_json = NULL;
	cJSON* chain_json = NULL;

	assert(data != NULL);

	data_json = cJSON_Parse(data);
	if(unlikely(data_json == NULL))
	{
		TFE_LOG_ERROR(g_default_logger, "Illegal JSON format: %s", data);
		goto error_out;
	}

	cert_json = cJSON_GetObjectItemCaseSensitive(data_json, "CERTIFICATE");
	key_json = cJSON_GetObjectItemCaseSensitive(data_json, "PRIVATE_KEY");
	chain_json = cJSON_GetObjectItemCaseSensitive(data_json, "CERTIFICATE_CHAIN");

	if(unlikely(cert_json == NULL))
	{
		TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No CERTIFICATE section: %s", data);
		goto error_out;
	}

	if(unlikely(key_json == NULL))
	{
		TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No PRIVATE_KEY section: %s", data);
		goto error_out;
	}

    if(unlikely(chain_json == NULL))
    {
        TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No CERTIFICATE_CHAIN section: %s", data);
        goto error_out;
    }

	if(unlikely(cert_json->valuestring == NULL))
	{
		TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No CERTIFICATE value: %s", data);
		goto error_out;
	}

	if(unlikely(key_json->valuestring == NULL))
	{
		TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, No PRIVATE_KEY value: %s", data);
		goto error_out;
	}

	cert = transform_cert_to_x509(cert_json->valuestring);
	if(unlikely(cert == NULL))
	{
		TFE_LOG_ERROR(g_default_logger, "Transform certificate to X509 failed: %s", cert_json->valuestring);
		goto error_out;
	}

	key = transform_key_to_EVP(key_json->valuestring);
	if(unlikely(key == NULL))
	{
		TFE_LOG_ERROR(g_default_logger, "Transform PRIVATE KEY to EVP failed: %s", key_json->valuestring);
		goto error_out;
	}

    chain = sk_X509_new_null();
	cJSON_ArrayForEach(chain_cert_json, chain_json)
	{
		if(unlikely(chain_cert_json->valuestring == NULL))
		{
			TFE_LOG_ERROR(g_default_logger, "Illegal JSON format, empty CERTIFICATE_CHAIN value.");
			goto error_out;
		}

		chain_cert = transform_cert_to_x509(chain_cert_json->valuestring);
		if(unlikely(chain_cert == NULL))
		{
			TFE_LOG_ERROR(g_default_logger, "Transform certificate chain entry to X509 failed: %s",
				chain_cert_json->valuestring); goto error_out;
		}

		sk_X509_push(chain, chain_cert);
	}

	_kyr= keyring_new(cert, key, chain);
	cJSON_Delete(data_json);
	return _kyr;

error_out:
	if(data_json!=NULL) cJSON_Delete(data_json);
	if(cert) X509_free(cert);
	if(key) EVP_PKEY_free(key);
	if(chain) sk_X509_pop_free(chain, X509_free);
	return NULL;
}

static long keyring_local_cache_query_cb(void * data, const uchar * key, uint size, void * user_arg)
{
	struct keyring_private* kyr=(struct keyring_private*)data;
	if(kyr == NULL)
	{
		return KEYRING_NOT_EXSITED;
	}
	else
	{
        struct promise *p = (struct promise *)user_arg;
        if (time(NULL) - kyr->update_time > ((struct key_keeper_promise_ctx *)promise_get_ctx(p))->ref_keeper->hash_expire_seconds)
        {
            return KEYRING_EXPIRE;
        }
		else
		{
			promise_success(p, data);
			return KEYRING_EXSITED;
		}
	}
}


static struct keyring_private* generate_x509_keyring(X509* origin_cert, X509* ca, EVP_PKEY* cakey, int cert_expire_time)
{
	//TODO: could be optimized to save cpu.

	EVP_PKEY* forge_key = ssl_key_genrsa(2048);
	X509* forge_cert = ssl_x509_forge(ca, cakey, origin_cert, forge_key, NULL, NULL, cert_expire_time);
	STACK_OF(X509)* chain = sk_X509_new_null();
	sk_X509_push(chain, ca);
	ssl_x509_refcount_inc(ca);
	ssl_x509_refcount_inc(forge_cert);
	struct keyring_private* _kyr= keyring_new(forge_cert, forge_key, chain);

	return _kyr;
}

static void certstore_rpc_on_succ(void* result, void* user)
{
	struct promise * p = (struct promise *) user;
	struct key_keeper_promise_ctx* ctx = (struct key_keeper_promise_ctx*)promise_get_ctx(p);
//	TFE_LOG_INFO(ctx->logger, "certstore rpc success");
	future_destroy(ctx->f_certstore_rpc);
	MESA_htable_handle htable= ctx->ref_keeper->cert_cache;
	const uchar* key = ctx->key;
	unsigned int key_len = ctx->key_len;
	struct tfe_rpc_response_result* response = tfe_rpc_release(result);
	int status_code = response->status_code;
	const char* status_msg = response->status_msg;

	if(status_code == HTTP_OK)
	{
		/* Copy a buffer ending with zero */
		char * data_as_string = (char *)malloc(response->len + 1);
		memcpy(data_as_string, response->data, response->len);
		data_as_string[response->len] = '\0';

		struct keyring_private* kyr= get_keyring_from_response(data_as_string);
		FREE(&data_as_string);

		if(kyr == NULL)
		{
			promise_failed(p, FUTURE_ERROR_EXCEPTION, "get_keyring_from_response failed");
			return;
		}

		if(!ctx->ref_keeper->no_cache)
		{
			keyring_ref_inc(kyr);
            kyr->update_time = time(NULL);
            pthread_rwlock_wrlock(&(ctx->ref_keeper->rwlock));
            if (MESA_htable_get_elem_num(ctx->ref_keeper->cert_cache) == ctx->ref_keeper->max_elem_num)
            {
                MESA_htable_del_oldest_manual(htable, NULL, ctx->ref_keeper->del_elem_num_once_when_full);
                TFE_LOG_DEBUG(ctx->ref_keeper->logger, "Key keeper cache full: %d del: %d left: %d", ctx->ref_keeper->max_elem_num, ctx->ref_keeper->del_elem_num_once_when_full, MESA_htable_get_elem_num(ctx->ref_keeper->cert_cache));
            }
            int ret = MESA_htable_add(htable, key, key_len, (void*)kyr);
            pthread_rwlock_unlock(&(ctx->ref_keeper->rwlock));
            if(ret<0)
			{
				key_keeper_free_keyring((struct keyring*)kyr);
			}
            else
            {
                TFE_LOG_DEBUG(ctx->ref_keeper->logger, "Key keeper cache add key: %s", ctx->key);
            }
        }
		ctx->ref_keeper->stat.new_issue++;
		promise_success(p, (void*)kyr);
		key_keeper_free_keyring((struct keyring*)kyr);
	}
	else
	{
		promise_failed(p, FUTURE_ERROR_EXCEPTION, status_msg);
	}
}

static void certstore_rpc_on_fail(enum e_future_error err, const char * what, void * user)
{
	struct promise * p = (struct promise *) user;
	struct key_keeper_promise_ctx* ctx= (struct key_keeper_promise_ctx*)promise_get_ctx(p);
	TFE_LOG_ERROR(ctx->logger, "certstore rpc failed, what is %s", what);
	future_destroy(ctx->f_certstore_rpc);
	promise_failed(p, err, what);
	//promise_dettach_ctx(p);
	//ctx_destroy_cb((void*)ctx);
}



static void key_keeper_free_serialized(void* data)
{
//	printf("call key_keeper_free_serialized\n");
	struct keyring_private* kyr = (struct keyring_private*)data;
	key_keeper_free_keyring(&(kyr->head));
}

static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int max_elem_num)
{
	UNUSED	int ret = 0;
	MESA_htable_handle htable = MESA_htable_born();
	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_SCREEN_PRINT_CTRL, 0);
    /*
	 * Use rwlock instead of htable mutex:
	 * MHO_THREAD_SAFE must be set to 0,
	 * MHO_MUTEX_NUM   must be set to 1,
	 * MHO_EXPIRE_TIME must be set to 0,
	 * otherwise MESA_htable_del_oldest_manual() execution error
	 */
    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 0);
    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 1);
    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, slot_size);
    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, max_elem_num);
    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, 0);
	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_ELIMIMINATE_TYPE,
		HASH_ELIMINATE_ALGO_FIFO);
	ret = __wrapper_MESA_htable_set_opt_func(htable, MHO_CBFUN_DATA_FREE,
	    (void *)key_keeper_free_serialized, sizeof(&key_keeper_free_serialized));
	//ret = __wrapper_MESA_htable_set_opt(htable, MHO_CBFUN_DATA_EXPIRE_NOTIFY,
	//	(void *)key_keeper_verify_cb);
	ret = MESA_htable_mature(htable);
	assert(ret == 0);
	return htable;
}

void key_keeper_destroy(struct key_keeper *keeper)
{
    if (!keeper->no_cache)
	{
    	pthread_rwlock_destroy(&(keeper->rwlock));
    	MESA_htable_destroy(keeper->cert_cache, NULL);
	}
	X509_free(keeper->trusted_ca_cert);
	EVP_PKEY_free(keeper->trusted_ca_key);

	X509_free(keeper->untrusted_ca_cert);
	EVP_PKEY_free(keeper->untrusted_ca_key);

	free(keeper);
	keeper = NULL;
	return;
}

struct evhttp_connection* key_keeper_evhttp_init(struct event_base * evbase, struct evdns_base* dnsbase, struct key_keeper * key_keeper_handler)
{	
	char *cert_store_host = key_keeper_handler->cert_store_host;
	unsigned int cert_store_port =key_keeper_handler->cert_store_port;
	return evhttp_connection_base_new(evbase, dnsbase, cert_store_host, cert_store_port);
}

static int health_check(struct key_keeper *keeper)
{
	char req_buff[1024] = { 0 };
	char rsp_buff[1024] = { 0 };
	struct sockaddr_in addr;
	char post_head[] =
    	"POST %s HTTP/1.1\r\n"
    	"Host: %s:%d\r\n"
    	"User-Agent: curl/7.47.0\r\n"
    	"Accept: */*\r\n"
    	"Content-Length: %d\r\n\r\n"
    	"%s\r\n";
	sprintf(req_buff, post_head, "/ca?health_check=1", keeper->cert_store_host, keeper->cert_store_port, strlen("health check"), "health check");

	int sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd == -1)
	{
		TFE_LOG_ERROR(keeper->logger, "CertStore health check thread fail to create socket(), %s", strerror(errno));
		/* after log, reset errno */
		errno = 0;
		goto error; 
	}
    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(keeper->cert_store_port);
    addr.sin_addr.s_addr = inet_addr(keeper->cert_store_host);
    if (connect(sockfd, (const struct sockaddr *)&addr, sizeof(addr)) == -1)
    {
        TFE_LOG_ERROR(keeper->logger, "CertStore health check thread fail to connect(), %s", strerror(errno));
		/* after log, reset errno */
		errno = 0;
		goto error; 
    }
	if (write(sockfd, req_buff, strlen(req_buff)) == -1)
	{
        TFE_LOG_ERROR(keeper->logger, "CertStore health check thread fail to write(), %s", strerror(errno));
		/* after log, reset errno */
		errno = 0;
		goto error; 
	}
	if (read(sockfd, rsp_buff, sizeof(rsp_buff)) == -1)
	{
        TFE_LOG_ERROR(keeper->logger, "CertStore health check thread fail to read(), %s", strerror(errno));
		/* after log, reset errno */
		errno = 0;
		goto error; 
	}
	close(sockfd);
	return 0;

error:
	if (sockfd)
		close(sockfd);
	return -1;
}
static void * certstore_health_check_thread(void * arg)
{
    struct key_keeper * keeper = (struct key_keeper *)arg;
	assert(keeper != NULL && keeper->thread == pthread_self());
	TFE_LOG_INFO(keeper->logger, "CertStore health check thread is running...");
	prctl(PR_SET_NAME, "CertStore_health_check");

	while (1)
	{
		if (health_check(keeper) == -1)
		{
			ATOMIC_INC(&certstore_is_unavailable);
			TFE_LOG_ERROR(keeper->logger, "CertStore health check thread on fail: certstore is unavailable !!!");
		}
		else
		{
			ATOMIC_ZERO(&certstore_is_unavailable);
		}
		sleep(1);
	}

	TFE_LOG_ERROR(keeper->logger, "CertStore health check thread is exit");

	return NULL;
}

struct key_keeper* key_keeper_init(const char * profile, const char* section, void* logger)
{
	struct key_keeper* keeper = ALLOC(struct key_keeper, 1);
	keeper->logger = logger;
	char tmp[TFE_STRING_MAX]={0};
	MESA_load_profile_string_def(profile, section, "mode", tmp, sizeof(tmp), "debug");
	if(strcasecmp(tmp, "debug") == 0)
	{
		keeper->work_mode = KK_MODE_LOCAL;
	}
	else
	{
		keeper->work_mode = KK_MODE_CERT_STORE;
	}

	MESA_load_profile_string_def(profile, section, "ca_path", keeper->trusted_ca_path,
		sizeof(keeper->trusted_ca_path), "./resource/tfe/mesalab-ca.pem");
	MESA_load_profile_string_def(profile, section, "untrusted_ca_path", keeper->untrusted_ca_path,
		sizeof(keeper->untrusted_ca_path), "./resource/tfe/mesalab-ca-untrust.pem");
	MESA_load_profile_string_def(profile, section, "cert_store_host", keeper->cert_store_host,
		sizeof(keeper->cert_store_host), "");

	MESA_load_profile_uint_def(profile, section, "enable_health_check", &(keeper->enable_health_check), 1);
	MESA_load_profile_uint_def(profile, section, "cert_store_port", &(keeper->cert_store_port), 80);
	MESA_load_profile_int_def(profile, section, "cert_expire_time", &(keeper->cert_expire_time), 24);	
    MESA_load_profile_uint_def(profile, section, "no_cache", &(keeper->no_cache), 0);
    if (!keeper->no_cache)
    {
        MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5 * 60);
        MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024 * 128);
        if (keeper->cert_expire_time != -1)
        {
            keeper->hash_expire_seconds = MIN(keeper->cert_expire_time * 1800, (int)(keeper->hash_expire_seconds));
        }

        keeper->max_elem_num = keeper->hash_slot_size * 4;
        keeper->del_elem_num_once_when_full = (keeper->max_elem_num / 10) > 0 ? (keeper->max_elem_num / 10) : 1;
        keeper->cert_cache = create_hash_table(keeper->hash_slot_size, keeper->max_elem_num);
        pthread_rwlock_init(&(keeper->rwlock), NULL);
    }
	if(0==strcmp(keeper->untrusted_ca_path, keeper->trusted_ca_path))
	{
		TFE_LOG_ERROR(logger, "Warnning: Trusted and Untrusted Root CA share the same path %s .", keeper->trusted_ca_path);
	}
	if(keeper->work_mode==KK_MODE_LOCAL)
	{
		keeper->trusted_ca_cert=ssl_x509_load(keeper->trusted_ca_path);
		keeper->trusted_ca_key=ssl_key_load(keeper->trusted_ca_path);
		if(keeper->trusted_ca_cert==NULL||keeper->trusted_ca_key==NULL)
		{
			TFE_LOG_ERROR(logger, "Load Trusted Root CA %s failed.", keeper->trusted_ca_path);
			goto error_out;
		}
		keeper->untrusted_ca_cert=ssl_x509_load(keeper->untrusted_ca_path);
		keeper->untrusted_ca_key=ssl_key_load(keeper->untrusted_ca_path);
		if(keeper->untrusted_ca_cert==NULL||keeper->trusted_ca_key==NULL)
		{
			TFE_LOG_ERROR(logger, "Load Untrusted Root CA %s failed.", keeper->untrusted_ca_path);
			goto error_out;
		}
	}
	// KK_MODE_CERT_STORE: create thread for certstore health check
	if (keeper->work_mode == KK_MODE_CERT_STORE && keeper->enable_health_check)
	{
		pthread_attr_t attr;
		pthread_attr_init(&attr);
		pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
		int ret = pthread_create(&keeper->thread, NULL, certstore_health_check_thread, (void *)keeper);
		if (unlikely(ret < 0))
		{
			TFE_LOG_ERROR(keeper->logger, "Failed at creating certstore health check thread: %s", strerror(errno));
			/* after log, reset errno */
			errno = 0;
			goto error_out;
		}
	}

    TFE_LOG_INFO(logger, "Key keeper cache, mode:%s, no_cache:%u, ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d, cert_expire_time:%d, max_elem_num:%d, del_elem_num_once_when_full:%d",
                 tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds, keeper->cert_expire_time, keeper->max_elem_num, keeper->del_elem_num_once_when_full);

    return keeper;

error_out:
	key_keeper_destroy(keeper);
	return NULL;

}

struct keyring* key_keeper_release_keyring(future_result_t* result)
{
	struct keyring_private* kyr=(struct keyring_private*)result;
	keyring_ref_inc(kyr);
	return &(kyr->head);
}

static uchar* get_key_by_cert(X509* cert, int keyring_id, unsigned int* len, int is_cert_valid)
{
	if(cert == NULL)
	{
		return NULL;
	}
	char* cert_fingerprint = NULL;
	cert_fingerprint = ssl_x509_fingerprint(cert, 0);
	if(cert_fingerprint == NULL)
	{
		return NULL;
	}
	char* key = ALLOC(char, HTABLE_MAX_KEY_LEN);
	memset(key, 0, HTABLE_MAX_KEY_LEN);
	snprintf(key, HTABLE_MAX_KEY_LEN, "%d:%d:", keyring_id, is_cert_valid);
	strncat(key, cert_fingerprint, HTABLE_MAX_KEY_LEN);
	*len = strnlen(key, HTABLE_MAX_KEY_LEN);
	free(cert_fingerprint);
	return (uchar*)key;
}


char* url_escape(char* url)
{
	if(url == NULL)
	{
		return NULL;
	}
	CURL *curl = curl_easy_init();
	char* _url = NULL;
	if(curl)
	{
		_url = curl_easy_escape(curl, url, strlen(url));
	}
	curl_easy_cleanup(curl);
	return _url;
}

void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const char* sni, int keyring_id, X509 * origin_cert, int is_cert_valid, struct event_base * evbase, struct evdns_base* dnsbase, struct evhttp_connection *evhttp)
{
	struct promise* p = future_to_promise(f);
	unsigned int len = 0;
	uchar* key = get_key_by_cert(origin_cert, keyring_id, &len, is_cert_valid);
	if(key == NULL)
	{
		promise_failed(p, FUTURE_ERROR_EXCEPTION, "get hash key by_cert failed");
		return;
	}
	struct key_keeper_promise_ctx* ctx = ALLOC(struct key_keeper_promise_ctx, 1);
	ctx->logger = keeper->logger;
	ctx->ref_keeper = keeper;
	ctx->key = key;
	ctx->key_len = len;
	promise_set_ctx(p, (void*)ctx, key_keeper_promise_free_ctx);
	long int cb_rtn = 0;
	keeper->stat.ask_times++;
	if(!keeper->no_cache)
	{
        char *tmp = tfe_strdup((const char *)ctx->key);
        pthread_rwlock_rdlock(&(keeper->rwlock));
        MESA_htable_search_cb(keeper->cert_cache, (const unsigned char*)(ctx->key), ctx->key_len, keyring_local_cache_query_cb, p, &cb_rtn);
        pthread_rwlock_unlock(&(keeper->rwlock));
        TFE_LOG_DEBUG(keeper->logger, "Key keeper cache search key: %s, found: %d (0:KEYRING_EXSITED, -1:KEYRING_NOT_EXSITED, -2:KEYRING_EXPIRE)", tmp, cb_rtn);
        free(tmp);
        if(cb_rtn == KEYRING_EXSITED)
		{
			//printf("KEYRING_EXSITED\n");
			return;
		}
        if (cb_rtn == KEYRING_EXPIRE)
		{
            pthread_rwlock_wrlock(&(keeper->rwlock));
            MESA_htable_del(keeper->cert_cache, (const unsigned char *)(ctx->key), ctx->key_len, NULL);
			pthread_rwlock_unlock(&(keeper->rwlock));
        }
    }
	switch(keeper->work_mode)
	{
		case KK_MODE_CERT_STORE:
        {
			char* origin_cert_pem = transform_cert_to_pem(origin_cert);
			if(origin_cert_pem == NULL)
			{
				promise_failed(p, FUTURE_ERROR_EXCEPTION, "transform origin_cert to pem failed");
				return;
			}
    		struct future* f_certstore_rpc = future_create("crt_store", certstore_rpc_on_succ, certstore_rpc_on_fail, p);
			ctx->f_certstore_rpc = f_certstore_rpc;
			char *url = NULL;

			//keyring_id = 1;
			if(sni == NULL || sni[0] == '\0')
			{
				asprintf(&url, "http://%s:%d/ca?keyring_id=%d&is_valid=%d",
					keeper->cert_store_host, keeper->cert_store_port, keyring_id, is_cert_valid);
			}
			else
			{
				asprintf(&url, "http://%s:%d/ca?keyring_id=%d&sni=%s&is_valid=%d",
					keeper->cert_store_host, keeper->cert_store_port, keyring_id, sni, is_cert_valid);
			}
			TFE_LOG_DEBUG(keeper->logger, "CertStore query: %.100s", url);
    		tfe_rpc_async_ask(f_certstore_rpc, url, POST, DONE_CB, origin_cert_pem, strlen(origin_cert_pem), evbase, dnsbase, evhttp);
			free(url);
			break;
        }
		case KK_MODE_LOCAL:
        {
			struct keyring_private* kyr=NULL;
			if(is_cert_valid == 1)
			{
				kyr=generate_x509_keyring(origin_cert, keeper->trusted_ca_cert, keeper->trusted_ca_key, keeper->cert_expire_time);
			}
			else
			{
				kyr=generate_x509_keyring(origin_cert, keeper->untrusted_ca_cert, keeper->untrusted_ca_key, keeper->cert_expire_time);
			}
			if(kyr)
			{
				if(!keeper->no_cache)
				{
					keyring_ref_inc(kyr);
                    kyr->update_time = time(NULL);
                    pthread_rwlock_wrlock(&(keeper->rwlock));
                    if (MESA_htable_get_elem_num(keeper->cert_cache) == keeper->max_elem_num)
                    {
                        MESA_htable_del_oldest_manual(keeper->cert_cache, NULL, keeper->del_elem_num_once_when_full);
                        TFE_LOG_DEBUG(keeper->logger, "Key keeper cache full: %d del: %d left: %d", keeper->max_elem_num, keeper->del_elem_num_once_when_full, MESA_htable_get_elem_num(keeper->cert_cache));
                    }
                    int ret = MESA_htable_add(ctx->ref_keeper->cert_cache, ctx->key, ctx->key_len, (void *)kyr);
                    pthread_rwlock_unlock(&(keeper->rwlock));
                    if(ret < 0)
					{
						key_keeper_free_keyring((struct keyring*)kyr);
					}
					else
					{
                        TFE_LOG_DEBUG(keeper->logger, "Key keeper cache add key: %s", ctx->key);
                    }		
				}
				promise_success(p, (void*)kyr);
				keeper->stat.new_issue++;
				key_keeper_free_keyring((struct keyring*)kyr);
			}
			else
			{
				promise_failed(p, FUTURE_ERROR_EXCEPTION, "generate X509 cert failed");
			}
			break;
        }
	}
	return;
}
void key_keeper_statistic(struct key_keeper *keeper, struct key_keeper_stat* result)
{
	if (!keeper->no_cache)
	{
    	// pthread_rwlock_rdlock(&(keeper->rwlock));
    	keeper->stat.cached_num=MESA_htable_get_elem_num(keeper->cert_cache);
    	// pthread_rwlock_unlock(&(keeper->rwlock));
	}
    *result = keeper->stat;
    return;
}