重构部分ssl_stream.cpp函数，支持记录ssl master key。

2018-10-18 12:13:41 +08:00
parent 71109517b4
commit f9b114a285
4 changed files with 180 additions and 43 deletions
--- a/platform/src/ssl_stream.cpp
+++ b/platform/src/ssl_stream.cpp
@@ -125,6 +125,10 @@ struct ssl_mgr
 	struct key_keeper * key_keeper;
 	struct event_base * ev_base_gc;
    struct event * gcev;
+
+	unsigned int log_master_key;
+	char master_key_file[TFE_PATH_MAX];
+	FILE* fp_master_key;
 	
 	void * logger;
 	screen_stat_handle_t fs_handle;
@@ -161,7 +165,7 @@ struct peek_client_hello_ctx
 	void * logger;
 };

-struct ssl_connect_origin_ctx
+struct ssl_connect_server_ctx
 {
 	struct bufferevent * bev;
 	struct ssl_stream * s_stream;
@@ -176,7 +180,7 @@ struct ssl_connect_origin_ctx
 	struct future * f_peek_chello;
 };

-struct ask_keyring_ctx
+struct ssl_connect_client_ctx
 {
 	int keyring_id;
 	struct ssl_stream * origin_ssl;
@@ -222,6 +226,10 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
 	{
 		FS_operate(mgr->fs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i])));
 	}
+	if(mgr->log_master_key && mgr->fp_master_key)
+	{
+		fflush(mgr->fp_master_key);
+	}
 	return;
 }
 void ssl_stat_init(struct ssl_mgr * mgr)
@@ -420,30 +428,51 @@ static int sslver_str2num(const char * version_str)

 	return sslversion;
 }
+static void log_ssl_master_key(SSL* ssl, int fd, tfe_conn_dir dir, FILE* fp)
+{
+	char* key_str=NULL;
+	key_str=ssl_ssl_masterkey_to_str(ssl);
+	char time_str[TFE_SYMBOL_MAX];
+	time_t now=time(NULL);
+	tfe_thread_safe_ctime(&now, time_str, sizeof(time_str));
+	struct tfe_stream_addr* addr=tfe_stream_addr_create_by_fd(fd, dir);
+	char* addr_string=tfe_stream_addr_to_str(addr);
+
+	fprintf(fp, "#%s %s %s\n%s\r", time_str, tfe_stream_conn_dir_to_str(dir), addr_string, key_str);
+
+	free(key_str);
+	tfe_stream_addr_free(addr);
+	free(addr_string);
+	return;
+}

 void ssl_manager_destroy(struct ssl_mgr * mgr)
 {
-	if (mgr->key_keeper != NULL)
+	if (mgr->key_keeper)
 	{
 		key_keeper_destroy(mgr->key_keeper);
 	}
-	if (mgr->trust_CA_store != NULL)
+	if (mgr->trust_CA_store)
 	{
 		X509_STORE_free(mgr->trust_CA_store);
 		mgr->trust_CA_store = NULL;
 	}
-	if(mgr->down_sess_cache!=NULL)
+	if(mgr->down_sess_cache)
 	{
 		ssl_sess_cache_destroy(mgr->down_sess_cache);
 	}
-	if(mgr->up_sess_cache!=NULL)
+	if(mgr->up_sess_cache)
 	{
 		ssl_sess_cache_destroy(mgr->up_sess_cache);
 	}
-	if(mgr->gcev!=NULL)
+	if(mgr->gcev)
 	{
 		 event_free(mgr->gcev);
 	}
+	if(mgr->fp_master_key)
+	{
+		fclose(mgr->fp_master_key);
+	}
 	free(mgr);
 }

@@ -543,6 +572,20 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
 	}

 	memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context));
+
+	MESA_load_profile_uint_def(ini_profile, section, "log_master_key", &(mgr->log_master_key), 0);
+	MESA_load_profile_string_def(ini_profile, section, "key_log_file", mgr->master_key_file, sizeof(mgr->master_key_file),
+		"./sslkeylog.log");
+	if(mgr->log_master_key)
+	{
+		mgr->fp_master_key=fopen(mgr->master_key_file, "a");
+		if(mgr->fp_master_key==NULL)
+		{
+			TFE_LOG_ERROR(logger, "Failed at open master key log file %s", mgr->master_key_file);
+			mgr->log_master_key=0;
+		}
+	}
+	
 	mgr->fs_handle=tfe_proxy_get_fs_handle();
 	ssl_stat_init(mgr);
 	
@@ -762,7 +805,7 @@ static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello *
 	return ssl;
 }

-void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx)
+void ssl_connect_server_ctx_free(struct ssl_connect_server_ctx * ctx)
 {
 	if (ctx->s_stream != NULL)
 	{
@@ -785,15 +828,15 @@ void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx)
 	return;
 }

-void ssl_connect_origin_ctx_free(void *p)
+void wrap_ssl_connect_server_ctx_free(void *p)
 {
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)p;
-	ssl_connect_origin_ctx_free(ctx);
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *)p;
+	ssl_connect_server_ctx_free(ctx);
 }

 struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result)
 {
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) result;
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
 	struct ssl_stream * ret = ctx->s_stream;
 	ctx->s_stream = NULL;                            //giveup ownership
 	return ret;
@@ -801,7 +844,7 @@ struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t *

 struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result)
 {
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) result;
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
 	struct bufferevent * ret = ctx->bev;
 	ctx->bev = NULL;                                //giveup ownership
 	return ret;
@@ -899,10 +942,10 @@ void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, void*
 * Callback for meta events on the up- and downstream connection bufferevents.
 * Called when EOF has been reached, a connection has been made, and on errors.
 */
-static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg)
+static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, void * arg)
 {
-	struct promise * promise = (struct promise *) arg;
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(promise);
+	struct promise * p = (struct promise *) arg;
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p);

 	struct ssl_stream * s_stream = ctx->s_stream;
 	struct ssl_mgr* mgr=s_stream->mgr;
@@ -912,17 +955,17 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v
 	{
 		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
 		ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, ctx->mgr->logger);
-		promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to original server failed.");
+		promise_failed(p, FUTURE_ERROR_EXCEPTION, "connect to original server failed.");
 	}
 	else if(events & BEV_EVENT_EOF)
 	{		
 		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
-		promise_failed(promise, FUTURE_ERROR_EXCEPTION, "original server closed.");
+		promise_failed(p, FUTURE_ERROR_EXCEPTION, "original server closed.");
 	}
 	else if(events & BEV_EVENT_TIMEOUT)
 	{
 		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
-		promise_failed(promise, FUTURE_ERROR_TIMEOUT, NULL);
+		promise_failed(p, FUTURE_ERROR_TIMEOUT, NULL);
 	}
 	else if(events & BEV_EVENT_CONNECTED)
 	{		
@@ -951,16 +994,20 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v
 			//Do not perform cert check on reused session.
 			s_stream->is_peer_cert_verify_passed=1;		
 		}
-		promise_success(promise, ctx);
+		if(mgr->log_master_key)
+		{
+			log_ssl_master_key(s_stream->ssl, ctx->fd_upstream, CONN_DIR_UPSTREAM, mgr->fp_master_key);
+		}
+		promise_success(p, ctx);
 	}
-	ssl_connect_origin_ctx_free(ctx);
+	wrap_ssl_connect_server_ctx_free(ctx);
 	return;
 }

 static void peek_chello_on_succ(future_result_t * result, void * user)
 {
 	struct promise * p = (struct promise *) user;
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_get_ctx(p);
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_get_ctx(p);

 	struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream.
 	if(chello->sni==NULL)
@@ -972,7 +1019,7 @@ static void peek_chello_on_succ(future_result_t * result, void * user)
 		ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS);

 	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1);
-	bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, p);
+	bufferevent_setcb(ctx->bev, NULL, NULL, ssl_server_connected_eventcb, p);
 	bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only

 	future_destroy(ctx->f_peek_chello);
@@ -983,10 +1030,10 @@ static void peek_chello_on_succ(future_result_t * result, void * user)
 static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user)
 {
 	struct promise * p = (struct promise *) user;
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p);
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p);
 	ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO]));
 	promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream.");
-	ssl_connect_origin_ctx_free(ctx);
+	wrap_ssl_connect_server_ctx_free(ctx);
 	return;
 }

@@ -994,7 +1041,7 @@ extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, e
 	evutil_socket_t fd_downstream, struct event_base * evbase)
 {
 	struct promise * p = future_to_promise(f);
-	struct ssl_connect_origin_ctx * ctx = ALLOC(struct ssl_connect_origin_ctx, 1);
+	struct ssl_connect_server_ctx * ctx = ALLOC(struct ssl_connect_server_ctx, 1);
 	int ret = 0;

 	ctx->addrlen = sizeof(ctx->addr);
@@ -1005,7 +1052,7 @@ extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, e
 	ctx->fd_upstream = fd_upstream;
 	ctx->evbase = evbase;
 	ctx->mgr = mgr;
-	promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free);
+	promise_set_ctx(p, ctx, wrap_ssl_connect_server_ctx_free);

 	ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p);
 	ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger);
@@ -1288,7 +1335,7 @@ static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt)
 	return ssl;
 }

-void ask_keyring_ctx_free(struct ask_keyring_ctx * ctx)
+void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx)
 {
 	X509_free(ctx->origin_crt);
 	ctx->origin_crt=NULL;
@@ -1312,31 +1359,70 @@ void ask_keyring_ctx_free(struct ask_keyring_ctx * ctx)
 	return;
 }

-void ask_keyring_ctx_free_cb(void * p)
+void wrap_ssl_connect_client_ctx_free(void * p)
 {
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)p;
-	ask_keyring_ctx_free(ctx);
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *)p;
+	ssl_connect_client_ctx_free(ctx);
 }

 struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result)
 {
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) result;
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result;
 	struct ssl_stream * ret = ctx->downstream;
 	ctx->downstream = NULL;
 	return ret;
 }
 struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result)
 {
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) result;
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result;
 	struct bufferevent * ret = ctx->bev_down;
 	ctx->bev_down = NULL;
 	return ret;
 }
+static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, void * arg)
+{
+	struct promise * p = (struct promise *) arg;
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p);
+
+	struct ssl_stream * s_stream = ctx->downstream;
+	struct ssl_mgr* mgr=s_stream->mgr;
+	SSL_SESSION * ssl_sess = NULL;
+
+	if (events & BEV_EVENT_ERROR)
+	{
+		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
+		ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, mgr->logger);
+		promise_failed(p, FUTURE_ERROR_EXCEPTION, "connect to client failed.");
+	}
+	else if(events & BEV_EVENT_EOF)
+	{		
+		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
+		promise_failed(p, FUTURE_ERROR_EXCEPTION, "client side closed.");
+	}
+	else if(events & BEV_EVENT_TIMEOUT)
+	{
+		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
+		promise_failed(p, FUTURE_ERROR_TIMEOUT, NULL);
+	}
+	else if(events & BEV_EVENT_CONNECTED)
+	{		
+		bufferevent_disable(ctx->bev_down, EV_READ | EV_WRITE);
+		bufferevent_setcb(ctx->bev_down, NULL, NULL, NULL, NULL); //leave a clean bev for on_success
+
+		if(mgr->log_master_key)
+		{
+			log_ssl_master_key(ctx->downstream->ssl, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key);
+		}
+		promise_success(p, ctx);
+	}
+	ssl_connect_client_ctx_free(ctx);
+	return;
+}

 void ask_keyring_on_succ(void * result, void * user)
 {
 	struct promise * p = (struct promise *) user;
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) promise_dettach_ctx(p);
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_get_ctx(p);

 	struct keyring * kyr = NULL;
 	struct ssl_mgr * mgr = ctx->ssl_mgr;
@@ -1344,24 +1430,27 @@ void ask_keyring_on_succ(void * result, void * user)


 	kyr = key_keeper_release_keyring(result);	//kyr will be freed at ssl downstream closing.
-	future_destroy(ctx->f_ask_keyring);
-	ctx->f_ask_keyring = NULL;
+

 	ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, kyr);
 	ctx->bev_down = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl,
 		BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS);
 	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1);
+	
+	bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p);
+	bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only
+
+	future_destroy(ctx->f_ask_keyring);
+	ctx->f_ask_keyring = NULL;

-	promise_success(p, ctx);
-	ask_keyring_ctx_free(ctx);
 }

 void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user)
 {
 	struct promise * p = (struct promise *) user;
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) promise_dettach_ctx(p);
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p);
 	promise_failed(p, error, what);
-	ask_keyring_ctx_free(ctx);
+	ssl_connect_client_ctx_free(ctx);
 	return;
 }

@@ -1375,7 +1464,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct
 	assert(upstream->dir == CONN_DIR_UPSTREAM);
 	int * previous_verfiy_result=NULL;
 	const char* sni=NULL;
-	struct ask_keyring_ctx * ctx = ALLOC(struct ask_keyring_ctx, 1);
+	struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1);
 	ctx->keyring_id = keyring_id;
 	ctx->ssl_mgr = mgr;
 	ctx->fd_downstream = fd_downstream;
@@ -1389,7 +1478,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct
 	}

 	struct promise * p = future_to_promise(f);
-	promise_set_ctx(p, ctx, ask_keyring_ctx_free_cb);
+	promise_set_ctx(p, ctx, wrap_ssl_connect_client_ctx_free);

 	ctx->f_ask_keyring = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p);
 	ctx->is_origin_crt_verify_passed = upstream->is_peer_cert_verify_passed;