重构部分ssl_stream.cpp函数，支持记录ssl master key。

common/include/tfe_utils.h

@@ -5,6 +5,7 @@
 
 #pragma once
 #include <MESA/MESA_handle_logger.h>
+#include <time.h>
 
 #define TFE_STRING_MAX			2048
 #define	TFE_PATH_MAX			256
@@ -91,6 +92,7 @@ do { MESA_handle_runtime_log(handler, RLOG_LV_DEBUG, "tfe", fmt, ##__VA_ARGS__);
 int addr_sock_to_layer(struct sockaddr * sock_addr, int sockaddrlen, struct layer_addr * layer_addr);
 int addr_layer_to_sock(struct layer_addr * layer_addr, struct sockaddr * sock_addr);
 char* tfe_strdup(const char* s);
+char *tfe_thread_safe_ctime(const time_t *tp, char *buf, int len);
 
 
 #define TFE_SET_USED(x)					(void)(x)

common/src/tfe_utils.cpp

@@ -3,6 +3,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <tfe_utils.h>
+#include <time.h>
 
 int addr_sock_to_layer(struct sockaddr * sock_addr, int sockaddrlen, struct layer_addr * layer_addr)
 {
@@ -28,4 +29,48 @@ char* tfe_strdup(const char* s)
 	memcpy(d,s,strlen(s)+1);
 	return d;
 }
+char *tfe_thread_safe_ctime(const time_t *tp, char *buf, int len)
+{
+    unsigned int year, month, day, weekday, hour, min, sec;
+    unsigned int year_days = 365;
+    unsigned int month_days[12] =
+    {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+    static unsigned char weekday_str[7][4] =
+    {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"};
+    static unsigned char month_str[12][4] = {"Jan", "Feb", "Mar", "Apr",
+                                             "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
+                                            };
+    sec = * tp;
+    min = sec / 60; sec = sec % 60;
+    hour = min / 60; min = min % 60; hour += 8;
+    day = hour / 24; hour = hour % 24;
+    weekday = day % 7; weekday = (weekday + 4) % 7;
+
+    for(year = 1970; day >= year_days;)
+    {
+        day -= year_days;
+        year ++;
+
+        if(0 == year % 4 && (0 != year % 100 || 0 == year % 400))
+            year_days = 366;
+        else year_days = 365;
+    }
+
+    if(366 == year_days) month_days[1] = 29;
+
+	//bug fix by yw 20120808
+    for(month = 0; day >= month_days[month];)
+    {
+        day -= month_days[month];
+        month ++;
+    }
+
+    /*
+    snprintf(buf, len, "%02d:%02d:%02d, %04d/%02d/%02d, %s",
+        hour, min, sec, year, month, day, weekday_str[week_day]);
+    */
+    snprintf(buf, len, "%s %s %d %02d:%02d:%02d %d", weekday_str[weekday],
+             month_str[month], day + 1, hour, min, sec, year);
+    return buf;
+}
 

platform/src/ssl_stream.cpp

@@ -125,6 +125,10 @@ struct ssl_mgr
 	struct key_keeper * key_keeper;
 	struct event_base * ev_base_gc;
     struct event * gcev;
+
+	unsigned int log_master_key;
+	char master_key_file[TFE_PATH_MAX];
+	FILE* fp_master_key;
 	
 	void * logger;
 	screen_stat_handle_t fs_handle;
@@ -161,7 +165,7 @@ struct peek_client_hello_ctx
 	void * logger;
 };
 
-struct ssl_connect_origin_ctx
+struct ssl_connect_server_ctx
 {
 	struct bufferevent * bev;
 	struct ssl_stream * s_stream;
@@ -176,7 +180,7 @@ struct ssl_connect_origin_ctx
 	struct future * f_peek_chello;
 };
 
-struct ask_keyring_ctx
+struct ssl_connect_client_ctx
 {
 	int keyring_id;
 	struct ssl_stream * origin_ssl;
@@ -222,6 +226,10 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
 	{
 		FS_operate(mgr->fs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i])));
 	}
+	if(mgr->log_master_key && mgr->fp_master_key)
+	{
+		fflush(mgr->fp_master_key);
+	}
 	return;
 }
 void ssl_stat_init(struct ssl_mgr * mgr)
@@ -420,30 +428,51 @@ static int sslver_str2num(const char * version_str)
 
 	return sslversion;
 }
+static void log_ssl_master_key(SSL* ssl, int fd, tfe_conn_dir dir, FILE* fp)
+{
+	char* key_str=NULL;
+	key_str=ssl_ssl_masterkey_to_str(ssl);
+	char time_str[TFE_SYMBOL_MAX];
+	time_t now=time(NULL);
+	tfe_thread_safe_ctime(&now, time_str, sizeof(time_str));
+	struct tfe_stream_addr* addr=tfe_stream_addr_create_by_fd(fd, dir);
+	char* addr_string=tfe_stream_addr_to_str(addr);
+
+	fprintf(fp, "#%s %s %s\n%s\r", time_str, tfe_stream_conn_dir_to_str(dir), addr_string, key_str);
+
+	free(key_str);
+	tfe_stream_addr_free(addr);
+	free(addr_string);
+	return;
+}
 
 void ssl_manager_destroy(struct ssl_mgr * mgr)
 {
-	if (mgr->key_keeper != NULL)
+	if (mgr->key_keeper)
 	{
 		key_keeper_destroy(mgr->key_keeper);
 	}
-	if (mgr->trust_CA_store != NULL)
+	if (mgr->trust_CA_store)
 	{
 		X509_STORE_free(mgr->trust_CA_store);
 		mgr->trust_CA_store = NULL;
 	}
-	if(mgr->down_sess_cache!=NULL)
+	if(mgr->down_sess_cache)
 	{
 		ssl_sess_cache_destroy(mgr->down_sess_cache);
 	}
-	if(mgr->up_sess_cache!=NULL)
+	if(mgr->up_sess_cache)
 	{
 		ssl_sess_cache_destroy(mgr->up_sess_cache);
 	}
-	if(mgr->gcev!=NULL)
+	if(mgr->gcev)
 	{
 		 event_free(mgr->gcev);
 	}
+	if(mgr->fp_master_key)
+	{
+		fclose(mgr->fp_master_key);
+	}
 	free(mgr);
 }
 
@@ -543,6 +572,20 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
 	}
 
 	memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context));
+
+	MESA_load_profile_uint_def(ini_profile, section, "log_master_key", &(mgr->log_master_key), 0);
+	MESA_load_profile_string_def(ini_profile, section, "key_log_file", mgr->master_key_file, sizeof(mgr->master_key_file),
+		"./sslkeylog.log");
+	if(mgr->log_master_key)
+	{
+		mgr->fp_master_key=fopen(mgr->master_key_file, "a");
+		if(mgr->fp_master_key==NULL)
+		{
+			TFE_LOG_ERROR(logger, "Failed at open master key log file %s", mgr->master_key_file);
+			mgr->log_master_key=0;
+		}
+	}
+	
 	mgr->fs_handle=tfe_proxy_get_fs_handle();
 	ssl_stat_init(mgr);
 	
@@ -762,7 +805,7 @@ static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello *
 	return ssl;
 }
 
-void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx)
+void ssl_connect_server_ctx_free(struct ssl_connect_server_ctx * ctx)
 {
 	if (ctx->s_stream != NULL)
 	{
@@ -785,15 +828,15 @@ void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx)
 	return;
 }
 
-void ssl_connect_origin_ctx_free(void *p)
+void wrap_ssl_connect_server_ctx_free(void *p)
 {
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)p;
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *)p;
-	ssl_connect_origin_ctx_free(ctx);
+	ssl_connect_server_ctx_free(ctx);
 }
 
 struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result)
 {
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) result;
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
 	struct ssl_stream * ret = ctx->s_stream;
 	ctx->s_stream = NULL;                            //giveup ownership
 	return ret;
@@ -801,7 +844,7 @@ struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t *
 
 struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result)
 {
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) result;
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
 	struct bufferevent * ret = ctx->bev;
 	ctx->bev = NULL;                                //giveup ownership
 	return ret;
@@ -899,10 +942,10 @@ void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, void*
  * Callback for meta events on the up- and downstream connection bufferevents.
  * Called when EOF has been reached, a connection has been made, and on errors.
  */
-static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg)
+static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, void * arg)
 {
-	struct promise * promise = (struct promise *) arg;
+	struct promise * p = (struct promise *) arg;
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(promise);
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p);
 
 	struct ssl_stream * s_stream = ctx->s_stream;
 	struct ssl_mgr* mgr=s_stream->mgr;
@@ -912,17 +955,17 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v
 	{
 		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
 		ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, ctx->mgr->logger);
-		promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to original server failed.");
+		promise_failed(p, FUTURE_ERROR_EXCEPTION, "connect to original server failed.");
 	}
 	else if(events & BEV_EVENT_EOF)
 	{		
 		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
-		promise_failed(promise, FUTURE_ERROR_EXCEPTION, "original server closed.");
+		promise_failed(p, FUTURE_ERROR_EXCEPTION, "original server closed.");
 	}
 	else if(events & BEV_EVENT_TIMEOUT)
 	{
 		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
-		promise_failed(promise, FUTURE_ERROR_TIMEOUT, NULL);
+		promise_failed(p, FUTURE_ERROR_TIMEOUT, NULL);
 	}
 	else if(events & BEV_EVENT_CONNECTED)
 	{		
@@ -951,16 +994,20 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v
 			//Do not perform cert check on reused session.
 			s_stream->is_peer_cert_verify_passed=1;		
 		}
-		promise_success(promise, ctx);
+		if(mgr->log_master_key)
+		{
+			log_ssl_master_key(s_stream->ssl, ctx->fd_upstream, CONN_DIR_UPSTREAM, mgr->fp_master_key);
+		}
+		promise_success(p, ctx);
 	}
-	ssl_connect_origin_ctx_free(ctx);
+	wrap_ssl_connect_server_ctx_free(ctx);
 	return;
 }
 
 static void peek_chello_on_succ(future_result_t * result, void * user)
 {
 	struct promise * p = (struct promise *) user;
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_get_ctx(p);
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_get_ctx(p);
 
 	struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream.
 	if(chello->sni==NULL)
@@ -972,7 +1019,7 @@ static void peek_chello_on_succ(future_result_t * result, void * user)
 		ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS);
 
 	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1);
-	bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, p);
+	bufferevent_setcb(ctx->bev, NULL, NULL, ssl_server_connected_eventcb, p);
 	bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only
 
 	future_destroy(ctx->f_peek_chello);
@@ -983,10 +1030,10 @@ static void peek_chello_on_succ(future_result_t * result, void * user)
 static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user)
 {
 	struct promise * p = (struct promise *) user;
-	struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p);
+	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p);
 	ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO]));
 	promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream.");
-	ssl_connect_origin_ctx_free(ctx);
+	wrap_ssl_connect_server_ctx_free(ctx);
 	return;
 }
 
@@ -994,7 +1041,7 @@ extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, e
 	evutil_socket_t fd_downstream, struct event_base * evbase)
 {
 	struct promise * p = future_to_promise(f);
-	struct ssl_connect_origin_ctx * ctx = ALLOC(struct ssl_connect_origin_ctx, 1);
+	struct ssl_connect_server_ctx * ctx = ALLOC(struct ssl_connect_server_ctx, 1);
 	int ret = 0;
 
 	ctx->addrlen = sizeof(ctx->addr);
@@ -1005,7 +1052,7 @@ extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, e
 	ctx->fd_upstream = fd_upstream;
 	ctx->evbase = evbase;
 	ctx->mgr = mgr;
-	promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free);
+	promise_set_ctx(p, ctx, wrap_ssl_connect_server_ctx_free);
 
 	ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p);
 	ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger);
@@ -1288,7 +1335,7 @@ static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt)
 	return ssl;
 }
 
-void ask_keyring_ctx_free(struct ask_keyring_ctx * ctx)
+void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx)
 {
 	X509_free(ctx->origin_crt);
 	ctx->origin_crt=NULL;
@@ -1312,31 +1359,70 @@ void ask_keyring_ctx_free(struct ask_keyring_ctx * ctx)
 	return;
 }
 
-void ask_keyring_ctx_free_cb(void * p)
+void wrap_ssl_connect_client_ctx_free(void * p)
 {
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)p;
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *)p;
-	ask_keyring_ctx_free(ctx);
+	ssl_connect_client_ctx_free(ctx);
 }
 
 struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result)
 {
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) result;
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result;
 	struct ssl_stream * ret = ctx->downstream;
 	ctx->downstream = NULL;
 	return ret;
 }
 struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result)
 {
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) result;
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result;
 	struct bufferevent * ret = ctx->bev_down;
 	ctx->bev_down = NULL;
 	return ret;
 }
+static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, void * arg)
+{
+	struct promise * p = (struct promise *) arg;
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p);
+
+	struct ssl_stream * s_stream = ctx->downstream;
+	struct ssl_mgr* mgr=s_stream->mgr;
+	SSL_SESSION * ssl_sess = NULL;
+
+	if (events & BEV_EVENT_ERROR)
+	{
+		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
+		ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, mgr->logger);
+		promise_failed(p, FUTURE_ERROR_EXCEPTION, "connect to client failed.");
+	}
+	else if(events & BEV_EVENT_EOF)
+	{		
+		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
+		promise_failed(p, FUTURE_ERROR_EXCEPTION, "client side closed.");
+	}
+	else if(events & BEV_EVENT_TIMEOUT)
+	{
+		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
+		promise_failed(p, FUTURE_ERROR_TIMEOUT, NULL);
+	}
+	else if(events & BEV_EVENT_CONNECTED)
+	{		
+		bufferevent_disable(ctx->bev_down, EV_READ | EV_WRITE);
+		bufferevent_setcb(ctx->bev_down, NULL, NULL, NULL, NULL); //leave a clean bev for on_success
+
+		if(mgr->log_master_key)
+		{
+			log_ssl_master_key(ctx->downstream->ssl, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key);
+		}
+		promise_success(p, ctx);
+	}
+	ssl_connect_client_ctx_free(ctx);
+	return;
+}
 
 void ask_keyring_on_succ(void * result, void * user)
 {
 	struct promise * p = (struct promise *) user;
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) promise_dettach_ctx(p);
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_get_ctx(p);
 
 	struct keyring * kyr = NULL;
 	struct ssl_mgr * mgr = ctx->ssl_mgr;
@@ -1344,24 +1430,27 @@ void ask_keyring_on_succ(void * result, void * user)
 
 
 	kyr = key_keeper_release_keyring(result);	//kyr will be freed at ssl downstream closing.
-	future_destroy(ctx->f_ask_keyring);
+
-	ctx->f_ask_keyring = NULL;
 
 	ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, kyr);
 	ctx->bev_down = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl,
 		BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS);
 	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1);
+	
+	bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p);
+	bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only
+
+	future_destroy(ctx->f_ask_keyring);
+	ctx->f_ask_keyring = NULL;
 
-	promise_success(p, ctx);
-	ask_keyring_ctx_free(ctx);
 }
 
 void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user)
 {
 	struct promise * p = (struct promise *) user;
-	struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) promise_dettach_ctx(p);
+	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p);
 	promise_failed(p, error, what);
-	ask_keyring_ctx_free(ctx);
+	ssl_connect_client_ctx_free(ctx);
 	return;
 }
 
@@ -1375,7 +1464,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct
 	assert(upstream->dir == CONN_DIR_UPSTREAM);
 	int * previous_verfiy_result=NULL;
 	const char* sni=NULL;
-	struct ask_keyring_ctx * ctx = ALLOC(struct ask_keyring_ctx, 1);
+	struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1);
 	ctx->keyring_id = keyring_id;
 	ctx->ssl_mgr = mgr;
 	ctx->fd_downstream = fd_downstream;
@@ -1389,7 +1478,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct
 	}
 
 	struct promise * p = future_to_promise(f);
-	promise_set_ctx(p, ctx, ask_keyring_ctx_free_cb);
+	promise_set_ctx(p, ctx, wrap_ssl_connect_client_ctx_free);
 
 	ctx->f_ask_keyring = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p);
 	ctx->is_origin_crt_verify_passed = upstream->is_peer_cert_verify_passed;

platform/src/ssl_utils.cc

@@ -302,7 +302,8 @@ char * ssl_ssl_masterkey_to_str(SSL * ssl)
 	unsigned char kbuf[48], rbuf[32];
 	k = &kbuf[0];
 	r = &rbuf[0];
-	SSL_SESSION_get_master_key(SSL_get0_session(ssl), k, sizeof(kbuf));
+	SSL_SESSION* sess=SSL_get0_session(ssl);
+	SSL_SESSION_get_master_key(sess, k, sizeof(kbuf));
 	SSL_get_client_random(ssl, r, sizeof(rbuf));
 
 	rv = asprintf(&str,