diff --git a/common/include/tfe_utils.h b/common/include/tfe_utils.h index 38a83cb..525ea6e 100644 --- a/common/include/tfe_utils.h +++ b/common/include/tfe_utils.h @@ -5,6 +5,7 @@ #pragma once #include +#include #define TFE_STRING_MAX 2048 #define TFE_PATH_MAX 256 @@ -91,6 +92,7 @@ do { MESA_handle_runtime_log(handler, RLOG_LV_DEBUG, "tfe", fmt, ##__VA_ARGS__); int addr_sock_to_layer(struct sockaddr * sock_addr, int sockaddrlen, struct layer_addr * layer_addr); int addr_layer_to_sock(struct layer_addr * layer_addr, struct sockaddr * sock_addr); char* tfe_strdup(const char* s); +char *tfe_thread_safe_ctime(const time_t *tp, char *buf, int len); #define TFE_SET_USED(x) (void)(x) diff --git a/common/src/tfe_utils.cpp b/common/src/tfe_utils.cpp index 5cf6234..7adfef7 100644 --- a/common/src/tfe_utils.cpp +++ b/common/src/tfe_utils.cpp @@ -3,6 +3,7 @@ #include #include #include +#include int addr_sock_to_layer(struct sockaddr * sock_addr, int sockaddrlen, struct layer_addr * layer_addr) { @@ -28,4 +29,48 @@ char* tfe_strdup(const char* s) memcpy(d,s,strlen(s)+1); return d; } +char *tfe_thread_safe_ctime(const time_t *tp, char *buf, int len) +{ + unsigned int year, month, day, weekday, hour, min, sec; + unsigned int year_days = 365; + unsigned int month_days[12] = + {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + static unsigned char weekday_str[7][4] = + {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; + static unsigned char month_str[12][4] = {"Jan", "Feb", "Mar", "Apr", + "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" + }; + sec = * tp; + min = sec / 60; sec = sec % 60; + hour = min / 60; min = min % 60; hour += 8; + day = hour / 24; hour = hour % 24; + weekday = day % 7; weekday = (weekday + 4) % 7; + + for(year = 1970; day >= year_days;) + { + day -= year_days; + year ++; + + if(0 == year % 4 && (0 != year % 100 || 0 == year % 400)) + year_days = 366; + else year_days = 365; + } + + if(366 == year_days) month_days[1] = 29; + + //bug fix by yw 20120808 + for(month = 0; day >= month_days[month];) + { + day -= month_days[month]; + month ++; + } + + /* + snprintf(buf, len, "%02d:%02d:%02d, %04d/%02d/%02d, %s", + hour, min, sec, year, month, day, weekday_str[week_day]); + */ + snprintf(buf, len, "%s %s %d %02d:%02d:%02d %d", weekday_str[weekday], + month_str[month], day + 1, hour, min, sec, year); + return buf; +} diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index bbca5a8..a1a376e 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -125,6 +125,10 @@ struct ssl_mgr struct key_keeper * key_keeper; struct event_base * ev_base_gc; struct event * gcev; + + unsigned int log_master_key; + char master_key_file[TFE_PATH_MAX]; + FILE* fp_master_key; void * logger; screen_stat_handle_t fs_handle; @@ -161,7 +165,7 @@ struct peek_client_hello_ctx void * logger; }; -struct ssl_connect_origin_ctx +struct ssl_connect_server_ctx { struct bufferevent * bev; struct ssl_stream * s_stream; @@ -176,7 +180,7 @@ struct ssl_connect_origin_ctx struct future * f_peek_chello; }; -struct ask_keyring_ctx +struct ssl_connect_client_ctx { int keyring_id; struct ssl_stream * origin_ssl; @@ -222,6 +226,10 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg) { FS_operate(mgr->fs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i]))); } + if(mgr->log_master_key && mgr->fp_master_key) + { + fflush(mgr->fp_master_key); + } return; } void ssl_stat_init(struct ssl_mgr * mgr) @@ -420,30 +428,51 @@ static int sslver_str2num(const char * version_str) return sslversion; } +static void log_ssl_master_key(SSL* ssl, int fd, tfe_conn_dir dir, FILE* fp) +{ + char* key_str=NULL; + key_str=ssl_ssl_masterkey_to_str(ssl); + char time_str[TFE_SYMBOL_MAX]; + time_t now=time(NULL); + tfe_thread_safe_ctime(&now, time_str, sizeof(time_str)); + struct tfe_stream_addr* addr=tfe_stream_addr_create_by_fd(fd, dir); + char* addr_string=tfe_stream_addr_to_str(addr); + + fprintf(fp, "#%s %s %s\n%s\r", time_str, tfe_stream_conn_dir_to_str(dir), addr_string, key_str); + + free(key_str); + tfe_stream_addr_free(addr); + free(addr_string); + return; +} void ssl_manager_destroy(struct ssl_mgr * mgr) { - if (mgr->key_keeper != NULL) + if (mgr->key_keeper) { key_keeper_destroy(mgr->key_keeper); } - if (mgr->trust_CA_store != NULL) + if (mgr->trust_CA_store) { X509_STORE_free(mgr->trust_CA_store); mgr->trust_CA_store = NULL; } - if(mgr->down_sess_cache!=NULL) + if(mgr->down_sess_cache) { ssl_sess_cache_destroy(mgr->down_sess_cache); } - if(mgr->up_sess_cache!=NULL) + if(mgr->up_sess_cache) { ssl_sess_cache_destroy(mgr->up_sess_cache); } - if(mgr->gcev!=NULL) + if(mgr->gcev) { event_free(mgr->gcev); } + if(mgr->fp_master_key) + { + fclose(mgr->fp_master_key); + } free(mgr); } @@ -543,6 +572,20 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section } memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context)); + + MESA_load_profile_uint_def(ini_profile, section, "log_master_key", &(mgr->log_master_key), 0); + MESA_load_profile_string_def(ini_profile, section, "key_log_file", mgr->master_key_file, sizeof(mgr->master_key_file), + "./sslkeylog.log"); + if(mgr->log_master_key) + { + mgr->fp_master_key=fopen(mgr->master_key_file, "a"); + if(mgr->fp_master_key==NULL) + { + TFE_LOG_ERROR(logger, "Failed at open master key log file %s", mgr->master_key_file); + mgr->log_master_key=0; + } + } + mgr->fs_handle=tfe_proxy_get_fs_handle(); ssl_stat_init(mgr); @@ -762,7 +805,7 @@ static SSL * upstream_ssl_create(struct ssl_mgr * mgr, const struct ssl_chello * return ssl; } -void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx) +void ssl_connect_server_ctx_free(struct ssl_connect_server_ctx * ctx) { if (ctx->s_stream != NULL) { @@ -785,15 +828,15 @@ void ssl_connect_origin_ctx_free(struct ssl_connect_origin_ctx * ctx) return; } -void ssl_connect_origin_ctx_free(void *p) +void wrap_ssl_connect_server_ctx_free(void *p) { - struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *)p; - ssl_connect_origin_ctx_free(ctx); + struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *)p; + ssl_connect_server_ctx_free(ctx); } struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result) { - struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) result; + struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result; struct ssl_stream * ret = ctx->s_stream; ctx->s_stream = NULL; //giveup ownership return ret; @@ -801,7 +844,7 @@ struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result) { - struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) result; + struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result; struct bufferevent * ret = ctx->bev; ctx->bev = NULL; //giveup ownership return ret; @@ -899,10 +942,10 @@ void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, void* * Callback for meta events on the up- and downstream connection bufferevents. * Called when EOF has been reached, a connection has been made, and on errors. */ -static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, void * arg) +static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, void * arg) { - struct promise * promise = (struct promise *) arg; - struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(promise); + struct promise * p = (struct promise *) arg; + struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p); struct ssl_stream * s_stream = ctx->s_stream; struct ssl_mgr* mgr=s_stream->mgr; @@ -912,17 +955,17 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, ctx->mgr->logger); - promise_failed(promise, FUTURE_ERROR_EXCEPTION, "connect to original server failed."); + promise_failed(p, FUTURE_ERROR_EXCEPTION, "connect to original server failed."); } else if(events & BEV_EVENT_EOF) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); - promise_failed(promise, FUTURE_ERROR_EXCEPTION, "original server closed."); + promise_failed(p, FUTURE_ERROR_EXCEPTION, "original server closed."); } else if(events & BEV_EVENT_TIMEOUT) { ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); - promise_failed(promise, FUTURE_ERROR_TIMEOUT, NULL); + promise_failed(p, FUTURE_ERROR_TIMEOUT, NULL); } else if(events & BEV_EVENT_CONNECTED) { @@ -951,16 +994,20 @@ static void ssl_connect_origin_eventcb(struct bufferevent * bev, short events, v //Do not perform cert check on reused session. s_stream->is_peer_cert_verify_passed=1; } - promise_success(promise, ctx); + if(mgr->log_master_key) + { + log_ssl_master_key(s_stream->ssl, ctx->fd_upstream, CONN_DIR_UPSTREAM, mgr->fp_master_key); + } + promise_success(p, ctx); } - ssl_connect_origin_ctx_free(ctx); + wrap_ssl_connect_server_ctx_free(ctx); return; } static void peek_chello_on_succ(future_result_t * result, void * user) { struct promise * p = (struct promise *) user; - struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_get_ctx(p); + struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_get_ctx(p); struct ssl_chello * chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream. if(chello->sni==NULL) @@ -972,7 +1019,7 @@ static void peek_chello_on_succ(future_result_t * result, void * user) ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1); - bufferevent_setcb(ctx->bev, NULL, NULL, ssl_connect_origin_eventcb, p); + bufferevent_setcb(ctx->bev, NULL, NULL, ssl_server_connected_eventcb, p); bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only future_destroy(ctx->f_peek_chello); @@ -983,10 +1030,10 @@ static void peek_chello_on_succ(future_result_t * result, void * user) static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user) { struct promise * p = (struct promise *) user; - struct ssl_connect_origin_ctx * ctx = (struct ssl_connect_origin_ctx *) promise_dettach_ctx(p); + struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p); ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO])); promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream."); - ssl_connect_origin_ctx_free(ctx); + wrap_ssl_connect_server_ctx_free(ctx); return; } @@ -994,7 +1041,7 @@ extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, e evutil_socket_t fd_downstream, struct event_base * evbase) { struct promise * p = future_to_promise(f); - struct ssl_connect_origin_ctx * ctx = ALLOC(struct ssl_connect_origin_ctx, 1); + struct ssl_connect_server_ctx * ctx = ALLOC(struct ssl_connect_server_ctx, 1); int ret = 0; ctx->addrlen = sizeof(ctx->addr); @@ -1005,7 +1052,7 @@ extern void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, e ctx->fd_upstream = fd_upstream; ctx->evbase = evbase; ctx->mgr = mgr; - promise_set_ctx(p, ctx, ssl_connect_origin_ctx_free); + promise_set_ctx(p, ctx, wrap_ssl_connect_server_ctx_free); ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p); ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); @@ -1288,7 +1335,7 @@ static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt) return ssl; } -void ask_keyring_ctx_free(struct ask_keyring_ctx * ctx) +void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx) { X509_free(ctx->origin_crt); ctx->origin_crt=NULL; @@ -1312,31 +1359,70 @@ void ask_keyring_ctx_free(struct ask_keyring_ctx * ctx) return; } -void ask_keyring_ctx_free_cb(void * p) +void wrap_ssl_connect_client_ctx_free(void * p) { - struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *)p; - ask_keyring_ctx_free(ctx); + struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *)p; + ssl_connect_client_ctx_free(ctx); } struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result) { - struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) result; + struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result; struct ssl_stream * ret = ctx->downstream; ctx->downstream = NULL; return ret; } struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result) { - struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) result; + struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result; struct bufferevent * ret = ctx->bev_down; ctx->bev_down = NULL; return ret; } +static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, void * arg) +{ + struct promise * p = (struct promise *) arg; + struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p); + + struct ssl_stream * s_stream = ctx->downstream; + struct ssl_mgr* mgr=s_stream->mgr; + SSL_SESSION * ssl_sess = NULL; + + if (events & BEV_EVENT_ERROR) + { + ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); + ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, mgr->logger); + promise_failed(p, FUTURE_ERROR_EXCEPTION, "connect to client failed."); + } + else if(events & BEV_EVENT_EOF) + { + ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); + promise_failed(p, FUTURE_ERROR_EXCEPTION, "client side closed."); + } + else if(events & BEV_EVENT_TIMEOUT) + { + ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); + promise_failed(p, FUTURE_ERROR_TIMEOUT, NULL); + } + else if(events & BEV_EVENT_CONNECTED) + { + bufferevent_disable(ctx->bev_down, EV_READ | EV_WRITE); + bufferevent_setcb(ctx->bev_down, NULL, NULL, NULL, NULL); //leave a clean bev for on_success + + if(mgr->log_master_key) + { + log_ssl_master_key(ctx->downstream->ssl, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key); + } + promise_success(p, ctx); + } + ssl_connect_client_ctx_free(ctx); + return; +} void ask_keyring_on_succ(void * result, void * user) { struct promise * p = (struct promise *) user; - struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) promise_dettach_ctx(p); + struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_get_ctx(p); struct keyring * kyr = NULL; struct ssl_mgr * mgr = ctx->ssl_mgr; @@ -1344,24 +1430,27 @@ void ask_keyring_on_succ(void * result, void * user) kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing. - future_destroy(ctx->f_ask_keyring); - ctx->f_ask_keyring = NULL; + ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, kyr); ctx->bev_down = bufferevent_openssl_socket_new(ctx->evbase, ctx->fd_downstream, ctx->downstream->ssl, BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); + + bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p); + bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only + + future_destroy(ctx->f_ask_keyring); + ctx->f_ask_keyring = NULL; - promise_success(p, ctx); - ask_keyring_ctx_free(ctx); } void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user) { struct promise * p = (struct promise *) user; - struct ask_keyring_ctx * ctx = (struct ask_keyring_ctx *) promise_dettach_ctx(p); + struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p); promise_failed(p, error, what); - ask_keyring_ctx_free(ctx); + ssl_connect_client_ctx_free(ctx); return; } @@ -1375,7 +1464,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct assert(upstream->dir == CONN_DIR_UPSTREAM); int * previous_verfiy_result=NULL; const char* sni=NULL; - struct ask_keyring_ctx * ctx = ALLOC(struct ask_keyring_ctx, 1); + struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1); ctx->keyring_id = keyring_id; ctx->ssl_mgr = mgr; ctx->fd_downstream = fd_downstream; @@ -1389,7 +1478,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct } struct promise * p = future_to_promise(f); - promise_set_ctx(p, ctx, ask_keyring_ctx_free_cb); + promise_set_ctx(p, ctx, wrap_ssl_connect_client_ctx_free); ctx->f_ask_keyring = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p); ctx->is_origin_crt_verify_passed = upstream->is_peer_cert_verify_passed; diff --git a/platform/src/ssl_utils.cc b/platform/src/ssl_utils.cc index df6a09d..53944ae 100644 --- a/platform/src/ssl_utils.cc +++ b/platform/src/ssl_utils.cc @@ -302,7 +302,8 @@ char * ssl_ssl_masterkey_to_str(SSL * ssl) unsigned char kbuf[48], rbuf[32]; k = &kbuf[0]; r = &rbuf[0]; - SSL_SESSION_get_master_key(SSL_get0_session(ssl), k, sizeof(kbuf)); + SSL_SESSION* sess=SSL_get0_session(ssl); + SSL_SESSION_get_master_key(sess, k, sizeof(kbuf)); SSL_get_client_random(ssl, r, sizeof(rbuf)); rv = asprintf(&str,