完成tfe中相关功能的开发,回归测试通过。

This commit is contained in:
zhengchao
2018-10-31 19:44:13 +08:00
parent 16935d273c
commit 198818a2aa
11 changed files with 384 additions and 92 deletions

View File

@@ -11,3 +11,9 @@ struct event_base * tfe_proxy_get_work_thread_evbase(unsigned int thread_id);
struct event_base * tfe_proxy_get_gc_evbase(void);
screen_stat_handle_t tfe_proxy_get_fs_handle(void);
int tfe_proxy_ssl_add_trust_ca(const char* pem_file);
int tfe_proxy_ssl_del_trust_ca(const char* pem_file);
int tfe_proxy_ssl_add_crl(const char* pem_file);
int tfe_proxy_ssl_del_crl(const char* pem_file);
void tfe_proxy_ssl_reset_trust_ca();

View File

@@ -1,4 +1,4 @@
add_executable(tfe src/key_keeper.cpp src/kni_acceptor.cpp src/ssl_stream.cpp src/ssl_sess_cache.cpp src/ssl_utils.cc src/tcp_stream.cpp src/main.cpp src/proxy.cpp)
add_executable(tfe src/key_keeper.cpp src/kni_acceptor.cpp src/ssl_stream.cpp src/ssl_sess_cache.cpp src/ssl_trusted_cert_storage.cpp src/ssl_utils.cc src/tcp_stream.cpp src/main.cpp src/proxy.cpp)
target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external)
target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal)

View File

@@ -0,0 +1,17 @@
#pragma once
#include <MESA/MESA_htable.h>
#include <assert.h>
inline int __wrapper_MESA_htable_set_opt_int(MESA_htable_handle table, enum MESA_htable_opt opt_type, unsigned int value)
{
int ret = MESA_htable_set_opt(table, opt_type, &value, (int)(sizeof(value)));
assert(ret == 0);
return ret;
}
inline int __wrapper_MESA_htable_set_opt_func(MESA_htable_handle table, enum MESA_htable_opt opt_type, void * val, size_t len)
{
int ret = MESA_htable_set_opt(table, opt_type, val, (int)len);
assert(ret == 0);
return ret;
}

View File

@@ -1,10 +1,7 @@
#pragma once
#include <ssl_utils.h>
#include <pthread.h>
#include <tfe_future.h>
#include <tfe_utils.h>
#include <MESA/MESA_htable.h>
#include <event2/event.h>
#include <openssl/ssl.h>
#include <tfe_future.h>
struct keyring
{

View File

@@ -1,12 +1,8 @@
#pragma once
#include <event2/event.h>
#include <tfe_future.h>
#include <tfe_types.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <openssl/x509.h>
#include <openssl/x509v3.h>
struct ssl_stream;
@@ -28,3 +24,11 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct
void ssl_stream_free_and_close_fd(struct ssl_stream * stream, struct event_base * evbase, evutil_socket_t fd);
void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, void* logger);
//Follow functions are allowed to call during runtime.
int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file);
int ssl_manager_del_trust_ca(struct ssl_mgr* mgr, const char* pem_file);
int ssl_manager_add_crl(struct ssl_mgr* mgr, const char* pem_file);
int ssl_manager_del_crl(struct ssl_mgr* mgr, const char* pem_file);
void ssl_manager_reset_trust_ca(struct ssl_mgr* mgr);

View File

@@ -0,0 +1,19 @@
#pragma once
#include <openssl/ssl.h>
enum ssl_X509_obj_type
{
SSL_X509_OBJ_CERT,
SSL_X509_OBJ_CRL
};
struct ssl_trusted_cert_storage;
struct ssl_trusted_cert_storage* ssl_trusted_cert_storage_create(const char* pem_bundle);
void ssl_trusted_cert_storage_destroy(struct ssl_trusted_cert_storage* storage);
int ssl_trusted_cert_storage_verify_conn(struct ssl_trusted_cert_storage* storage, const SSL * ssl, char* reason, size_t n_reason);
int ssl_trusted_cert_storage_add(struct ssl_trusted_cert_storage* storage, enum ssl_X509_obj_type type, const char* filename);
int ssl_trusted_cert_storage_del(struct ssl_trusted_cert_storage* storage, enum ssl_X509_obj_type type, const char* filename);
void ssl_trusted_cert_storage_reset(struct ssl_trusted_cert_storage* storage);

View File

@@ -1,10 +1,18 @@
#include "key_keeper.h"
#include "MESA_htable_aux.h"
#include <pthread.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include "MESA/MESA_prof_load.h"
#include "tfe_rpc.h"
#include <MESA/MESA_prof_load.h>
#include <MESA/MESA_htable.h>
#include <ssl_utils.h>
#include <tfe_rpc.h>
#include <tfe_utils.h>
#include <event2/http.h>
#include <cjson/cJSON.h>
#include <curl/curl.h>
@@ -440,22 +448,7 @@ static void certstore_rpc_on_fail(enum e_future_error err, const char * what, vo
//ctx_destroy_cb((void*)ctx);
}
/*
* Certificate, including private key and keyring chain.
*/
static int __wrapper_MESA_htable_set_opt_int(MESA_htable_handle table, enum MESA_htable_opt opt_type, unsigned int value)
{
int ret = MESA_htable_set_opt(table, opt_type, &value, (int)(sizeof(value)));
assert(ret == 0);
return ret;
}
static int __wrapper_MESA_htable_set_opt_func(MESA_htable_handle table, enum MESA_htable_opt opt_type, void * val, size_t len)
{
int ret = MESA_htable_set_opt(table, opt_type, val, (int)len);
assert(ret == 0);
return ret;
}
static void key_keeper_free_serialized(void* data)
{

View File

@@ -382,5 +382,27 @@ screen_stat_handle_t tfe_proxy_get_fs_handle(void)
{
return g_default_proxy->fs_handle;
}
int tfe_proxy_ssl_add_trust_ca(const char* pem_file)
{
return ssl_manager_add_trust_ca(g_default_proxy->ssl_mgr_handler, pem_file);
}
int tfe_proxy_ssl_del_trust_ca(const char* pem_file)
{
return ssl_manager_del_trust_ca(g_default_proxy->ssl_mgr_handler, pem_file);
}
int tfe_proxy_ssl_add_crl(const char* pem_file)
{
return ssl_manager_add_crl(g_default_proxy->ssl_mgr_handler, pem_file);
}
int tfe_proxy_ssl_del_crl(const char* pem_file)
{
return ssl_manager_del_crl(g_default_proxy->ssl_mgr_handler, pem_file);
}
void tfe_proxy_ssl_reset_trust_ca(void)
{
ssl_manager_reset_trust_ca(g_default_proxy->ssl_mgr_handler);
return;
}

View File

@@ -21,6 +21,13 @@
#include <event2/thread.h>
#include <event2/dns.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <openssl/x509.h>
#include <openssl/x509v3.h>
#include <MESA/MESA_htable.h>
#include <MESA/MESA_prof_load.h>
@@ -30,6 +37,7 @@
#include <tfe_proxy.h>
#include <key_keeper.h>
#include <ssl_sess_cache.h>
#include <ssl_trusted_cert_storage.h>
#include <ssl_utils.h>
#include <platform.h>
@@ -124,8 +132,7 @@ struct ssl_mgr
uint8_t ssl_mode_release_buffers;
char trust_CA_file[TFE_PATH_MAX];
char trust_CA_dir[TFE_PATH_MAX];
X509_STORE * trust_CA_store;
struct ssl_trusted_cert_storage * trust_CA_store;
struct key_keeper * key_keeper;
struct event_base * ev_base_gc;
struct event * gcev;
@@ -462,7 +469,7 @@ void ssl_manager_destroy(struct ssl_mgr * mgr)
}
if (mgr->trust_CA_store)
{
X509_STORE_free(mgr->trust_CA_store);
ssl_trusted_cert_storage_destroy(mgr->trust_CA_store);
mgr->trust_CA_store = NULL;
}
if(mgr->down_sess_cache)
@@ -552,34 +559,17 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
TFE_LOG_ERROR(logger, "Certificate Manager initiate failed.");
goto error_out;
}
mgr->trust_CA_store = X509_STORE_new();
MESA_load_profile_string_def(ini_profile, section, "trust_CA_file", mgr->trust_CA_file, sizeof(mgr->trust_CA_file),
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem");
mgr->trust_CA_store = ssl_trusted_cert_storage_create(mgr->trust_CA_file);
if (mgr->trust_CA_store == NULL)
{
TFE_LOG_ERROR(logger, "Failed at creating X509_STORE");
goto error_out;
}
ret = X509_STORE_set_default_paths(mgr->trust_CA_store);
if (ret == 0)
{
TFE_LOG_ERROR(logger, "Failed at setting default paths for X509_STORE.");
goto error_out;
}
MESA_load_profile_string_def(ini_profile, section, "trust_CA_file", mgr->trust_CA_file, sizeof(mgr->trust_CA_file),
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem");
MESA_load_profile_string_def(ini_profile, section, "trust_CA_dir", mgr->trust_CA_dir, sizeof(mgr->trust_CA_dir),
"");
ret = X509_STORE_load_locations(mgr->trust_CA_store, strlen(mgr->trust_CA_file) > 0 ? mgr->trust_CA_file : NULL,
strlen(mgr->trust_CA_dir) > 0 ? mgr->trust_CA_dir : NULL);
if (ret == 0)
{
TFE_LOG_ERROR(logger, "Failed at setting load locations for X509_STORE");
goto error_out;
}
memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context));
@@ -608,39 +598,6 @@ error_out:
return NULL;
}
int ssl_conn_verify_cert(X509_STORE * store, const SSL * ssl, char** error_string)
{
int ret = 0, err_code=0;
char *subj=NULL, *issuer=NULL;
STACK_OF(X509) * cert_chain = SSL_get_peer_cert_chain(ssl);
if (cert_chain == NULL)
{
// The peer certificate chain is not necessarily available after reusing a session, in which case a NULL pointer is returned.
return 1;
}
X509_STORE_CTX * ctx = X509_STORE_CTX_new();
X509 * cert = sk_X509_value(cert_chain, 0);
ret = X509_STORE_CTX_init(ctx, store, cert, cert_chain);
assert(ret == 1);
//If a complete chain can be built and validated this function returns 1, otherwise it return zero or negtive code.
ret = X509_verify_cert(ctx);
if(ret!=1)
{
err_code=X509_STORE_CTX_get_error(ctx);
subj=ssl_x509_subject(cert);
issuer=ssl_x509_issuer(cert);
asprintf(error_string, "%s : subject - %s issuer - %s"
, X509_verify_cert_error_string(err_code)
, subj
, issuer);
free(subj);
free(issuer);
}
X509_STORE_CTX_free(ctx);
return (ret == 1);
}
void peek_client_hello_ctx_free(struct peek_client_hello_ctx * _ctx)
{
event_free(_ctx->ev);
@@ -975,7 +932,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
struct ssl_stream * s_stream = ctx->s_stream;
struct ssl_mgr* mgr=s_stream->mgr;
SSL_SESSION * ssl_sess = NULL;
char* error_string=NULL;
char error_string[TFE_STRING_MAX];
if (events & BEV_EVENT_ERROR)
{
@@ -1000,7 +957,8 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
if(!SSL_session_reused(s_stream->ssl))
{
s_stream->is_peer_cert_verify_passed = ssl_conn_verify_cert(s_stream->mgr->trust_CA_store, s_stream->ssl, &error_string);
s_stream->is_peer_cert_verify_passed = ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store,
s_stream->ssl, error_string, sizeof(error_string));
if(s_stream->is_peer_cert_verify_passed)
{
//ONLY verified session is cacheable.
@@ -1015,7 +973,6 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
char* addr_str=tfe_string_addr_create_by_fd(ctx->fd_upstream, CONN_DIR_UPSTREAM);
TFE_LOG_INFO(mgr->logger, "Fake Cert %s %s : %s", addr_str, ctx->s_stream->client_hello->sni, error_string);
free(addr_str);
free(error_string);
}
}
else
@@ -1084,7 +1041,7 @@ static void peek_chello_on_fail(enum e_future_error err, const char * what, void
return;
}
extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream,
void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream,
evutil_socket_t fd_downstream, struct event_base * evbase)
{
struct promise * p = future_to_promise(f);
@@ -1535,7 +1492,7 @@ void ask_keyring_on_fail(enum e_future_error error, const char * what, void * us
/*
* Create a SSL stream for the incoming connection, based on the upstream.
*/
extern void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream,
void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream,
evutil_socket_t fd_downstream, int keyring_id, struct event_base * evbase)
{
@@ -1710,3 +1667,26 @@ void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_bas
sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase);
pxy_ssl_shutdown_cb(fd, 0, sslshutctx);
}
int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file)
{
return ssl_trusted_cert_storage_add(mgr->trust_CA_store, SSL_X509_OBJ_CERT, pem_file);
}
int ssl_manager_del_trust_ca(struct ssl_mgr* mgr, const char* pem_file)
{
return ssl_trusted_cert_storage_del(mgr->trust_CA_store, SSL_X509_OBJ_CERT, pem_file);
}
int ssl_manager_add_crl(struct ssl_mgr* mgr, const char* pem_file)
{
return ssl_trusted_cert_storage_add(mgr->trust_CA_store, SSL_X509_OBJ_CRL, pem_file);
}
int ssl_manager_del_crl(struct ssl_mgr* mgr, const char* pem_file)
{
return ssl_trusted_cert_storage_del(mgr->trust_CA_store, SSL_X509_OBJ_CRL, pem_file);
}
void ssl_manager_reset_trust_ca(struct ssl_mgr* mgr)
{
ssl_trusted_cert_storage_reset(mgr->trust_CA_store);
return;
}

View File

@@ -0,0 +1,254 @@
#include "ssl_trusted_cert_storage.h"
#include "MESA_htable_aux.h"
#include <MESA/MESA_htable.h>
#include <ssl_utils.h>
#include <tfe_utils.h>
#include <pthread.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
struct ssl_X509_object
{
char* filename;
enum ssl_X509_obj_type type;
};
static void free_ssl_x509_obj(void* data)
{
struct ssl_X509_object* obj=(struct ssl_X509_object*)data;
free(obj->filename);
free(obj);
return;
}
struct ssl_trusted_cert_storage
{
char* pem_bundle;
MESA_htable_handle hash_table;
pthread_rwlock_t rwlock;
X509_STORE* effective_store;
};
static X509_STORE* _X509_store_create(const char* pem_bundle)
{
int ret=0;
X509_STORE* store=X509_STORE_new();
if (store == NULL)
{
return NULL;
}
ret = X509_STORE_set_default_paths(store);
if (ret == 0)
{
return NULL;
}
ret = X509_STORE_load_locations(store, pem_bundle, NULL);
if (ret == 0)
{
return NULL;
}
return store;
}
static MESA_htable_handle _create_mesa_htable(void)
{
int ret=0;
MESA_htable_handle htable = MESA_htable_born();
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_SCREEN_PRINT_CTRL, 0);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 0);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 16);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, 1024*4);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, 0);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, 0);
ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_ELIMIMINATE_TYPE,
HASH_ELIMINATE_ALGO_FIFO);
ret = __wrapper_MESA_htable_set_opt_func(htable, MHO_CBFUN_DATA_FREE,
(void *)free_ssl_x509_obj, sizeof(&free_ssl_x509_obj));
//ret = __wrapper_MESA_htable_set_opt(htable, MHO_CBFUN_DATA_EXPIRE_NOTIFY,
// (void *)key_keeper_verify_cb);
ret = MESA_htable_mature(htable);
return htable;
}
struct ssl_trusted_cert_storage* ssl_trusted_cert_storage_create(const char* pem_bundle)
{
int ret=0;
struct ssl_trusted_cert_storage* storage=ALLOC(struct ssl_trusted_cert_storage, 1);
storage->effective_store=_X509_store_create(pem_bundle);
if (storage->effective_store == NULL)
{
return NULL;
}
storage->pem_bundle=tfe_strdup(pem_bundle);
storage->hash_table=_create_mesa_htable();
pthread_rwlock_init(&(storage->rwlock), NULL);
return storage;
}
void ssl_trusted_cert_storage_destroy(struct ssl_trusted_cert_storage* storage)
{
return;
}
static int _X509_add_cert_or_crl_add(X509_STORE* store, enum ssl_X509_obj_type type, const char* filename)
{
int ret=0;
BIO *bio=NULL;
X509* x=NULL;
X509_CRL* x_crl=NULL;
bio=BIO_new_file(filename, "r");
if(bio==NULL)
{
return -1;
}
ret=0;
if(type==SSL_X509_OBJ_CERT)
{
x=PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL);
if(x!=NULL)
{
ret=X509_STORE_add_cert(store, x);
if(ret==0)
{
X509_free(x);
}
}
}
else if(type==SSL_X509_OBJ_CRL)
{
x_crl=PEM_read_bio_X509_CRL(bio, NULL, NULL, NULL);
if(x_crl!=NULL)
{
ret=X509_STORE_add_crl(store, x_crl);
if(ret==0)
{
X509_CRL_free(x_crl);
}
}
}
if(ret==0)
{
BIO_free(bio);
return -1;
}
BIO_free(bio);
return 1;
}
int ssl_trusted_cert_storage_add(struct ssl_trusted_cert_storage* storage, enum ssl_X509_obj_type type, const char* filename)
{
int ret=0;
struct ssl_X509_object* obj=NULL;
void* data=NULL;
pthread_rwlock_wrlock(&(storage->rwlock));
data=MESA_htable_search(storage->hash_table, (const unsigned char*)filename, strlen(filename));
if(data!=NULL)//duplicated
{
ret=-1;
goto error_out;
}
ret=_X509_add_cert_or_crl_add(storage->effective_store, type, filename);
if(ret<0)
{
ret=-1;
goto error_out;
}
obj=ALLOC(struct ssl_X509_object, 1);
obj->type=SSL_X509_OBJ_CERT;
obj->filename=tfe_strdup(filename);
ret=MESA_htable_add(storage->hash_table, (const unsigned char*)obj->filename, strlen(obj->filename), obj);
assert(ret>0);
ret=1;
error_out:
pthread_rwlock_unlock(&(storage->rwlock));
return ret;
}
static void cert_storage_htable_traverse_cb(const uchar * key, uint size, void * data, void * user)
{
X509_STORE* store=(X509_STORE* )user;
struct ssl_X509_object* obj=(struct ssl_X509_object*)data;
_X509_add_cert_or_crl_add(store, obj->type, obj->filename);
return;
}
int ssl_trusted_cert_storage_del(struct ssl_trusted_cert_storage* storage, enum ssl_X509_obj_type type, const char* filename)
{
int ret=0;
X509_STORE* temp_store=NULL;
pthread_rwlock_wrlock(&(storage->rwlock));
ret=MESA_htable_del(storage->hash_table, (const unsigned char*)filename, strlen(filename), NULL);
if(ret<0)
{
ret=-1;
goto error_out;
}
temp_store=_X509_store_create(storage->pem_bundle);
MESA_htable_iterate(storage->hash_table, cert_storage_htable_traverse_cb, temp_store);
X509_STORE_free(storage->effective_store);
storage->effective_store=temp_store;
ret=1;
error_out:
pthread_rwlock_unlock(&(storage->rwlock));
return ret;
}
void ssl_trusted_cert_storage_reset(struct ssl_trusted_cert_storage* storage)
{
X509_STORE* temp_store=NULL;
MESA_htable_destroy(storage->hash_table, NULL);
storage->hash_table=_create_mesa_htable();
temp_store=_X509_store_create(storage->pem_bundle);
pthread_rwlock_wrlock(&(storage->rwlock));
X509_STORE_free(storage->effective_store);
storage->effective_store=temp_store;
pthread_rwlock_unlock(&(storage->rwlock));
return;
}
int ssl_trusted_cert_storage_verify_conn(struct ssl_trusted_cert_storage* storage, const SSL * ssl, char* reason, size_t n_reason)
{
int ret = 0, err_code=0;
char *subj=NULL, *issuer=NULL;
STACK_OF(X509) * cert_chain = SSL_get_peer_cert_chain(ssl);
if (cert_chain == NULL)
{
// The peer certificate chain is not necessarily available after reusing a session, in which case a NULL pointer is returned.
return 1;
}
pthread_rwlock_rdlock(&(storage->rwlock));
X509_STORE_CTX * ctx = X509_STORE_CTX_new();
X509 * cert = sk_X509_value(cert_chain, 0);
ret = X509_STORE_CTX_init(ctx, storage->effective_store, cert, cert_chain);
assert(ret == 1);
//If a complete chain can be built and validated this function returns 1, otherwise it return zero or negtive code.
ret = X509_verify_cert(ctx);
if(ret!=1)
{
err_code=X509_STORE_CTX_get_error(ctx);
subj=ssl_x509_subject(cert);
issuer=ssl_x509_issuer(cert);
snprintf(reason, n_reason, "%s : subject - %s issuer - %s"
, X509_verify_cert_error_string(err_code)
, subj
, issuer);
free(subj);
free(issuer);
}
X509_STORE_CTX_free(ctx);
pthread_rwlock_unlock(&(storage->rwlock));
return (ret == 1);
}

View File

@@ -19,7 +19,7 @@ int main()
future_promise_library_init();
//struct event_base* evbase = event_base_new();
struct key_keeper * keeper = key_keeper_init("./conf/tfe.conf", "key_keeper", logger);
struct key_keeper_ctx* ctx = ALLOC(struct key_keeper_ctx, 1);
struct key_keeper_ctx* ctx = (struct key_keeper_ctx*)malloc(sizeof(struct key_keeper_ctx));
ctx->keeper = keeper;
struct future* f = future_create("key_keeper", ask_key_keeper_on_succ, ask_key_keeper_on_fail, ctx);
ctx->f = f;