gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop-tfe4a' of https://git.mesalab.cn/tango/tfe into develop-tfe4a

Browse Source
This commit is contained in:
luwenpeng
2019-09-05 18:02:41 +08:00
parent 6b4a45c005 1484f763f6
commit ff19c00dc1
8 changed files with 177 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ci/travis.sh
View File

@@ -51,5 +51,5 @@ if [ -n "${PACKAGE}" ]; then
fi fi
if [ -n "${UPLOAD_SYMBOL_FILES}" ]; then if [ -n "${UPLOAD_SYMBOL_FILES}" ]; then
sentry-cli upload-dif -t elf build/ sentry-cli upload-dif -t elf ./
fi fi

6
platform/include/internal/ssl_stream_core.h
View File

@@ -15,7 +15,7 @@ enum ssl_stream_error
SSL_STREAM_R_CLIENT_PROTOCOL_ERROR, SSL_STREAM_R_CLIENT_PROTOCOL_ERROR,
__SSL_STREAM_R_MAX __SSL_STREAM_R_MAX
}; };
struct ssl_mgr; struct ssl_mgr;
@@ -28,8 +28,6 @@ void ssl_stream_process_zero_eof(struct ssl_stream * s_stream, struct ssl_mgr* m
enum ssl_stream_action ssl_upstream_create_result_release_action(future_result_t * result); enum ssl_stream_action ssl_upstream_create_result_release_action(future_result_t * result);
evutil_socket_t ssl_upstream_create_result_release_fd(future_result_t * result);
struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result); struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result);
struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result); struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result);
void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream, void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream,
@@ -44,7 +42,7 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct
evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream); evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream);
void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev); void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev);
const char* ssl_stream_dump_info(struct ssl_stream *stream, char* buffer, size_t sz); const char* ssl_stream_dump_info(struct ssl_stream *stream, char* buffer, size_t sz);
//Follow functions are allowed to call during runtime. //Follow functions are allowed to call during runtime.
int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file); int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file);

28
platform/src/acceptor_kni_v2.cpp
View File

@@ -47,6 +47,8 @@ void acceptor_kni_v2_event(evutil_socket_t fd, short what, void * user)
struct tfe_cmsg * cmsg = NULL; struct tfe_cmsg * cmsg = NULL;
int * __fds = NULL; int * __fds = NULL;
unsigned int __nr_fds = 0;
assert(__ctx != NULL && __ctx->thread == pthread_self()); assert(__ctx != NULL && __ctx->thread == pthread_self());
assert(what & EV_READ); assert(what & EV_READ);
@@ -91,11 +93,19 @@ void acceptor_kni_v2_event(evutil_socket_t fd, short what, void * user)
TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_FIRSTHDR() from incoming fds."); TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_FIRSTHDR() from incoming fds.");
goto __drop_recieved_fds; goto __drop_recieved_fds;
} }
__fds = (int *) (CMSG_DATA(__cmsghdr)); switch(__cmsghdr->cmsg_len)
if (unlikely(__fds == NULL))
{ {
TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_DATA() from incoming fds."); case CMSG_LEN(0 * sizeof(int)): { __nr_fds = 0; break;}
case CMSG_LEN(1 * sizeof(int)): { __nr_fds = 1; break;}
case CMSG_LEN(2 * sizeof(int)): { __nr_fds = 2; break; }
default: assert(0);
}
__fds = (int *) (CMSG_DATA(__cmsghdr));
if (unlikely(__fds == NULL || __nr_fds < 2))
{
TFE_LOG_ERROR(__ctx->logger, "No available file descriptors, drop the incoming fds.");
goto __drop_recieved_fds; goto __drop_recieved_fds;
} }
@@ -120,9 +130,13 @@ __die:
return; return;
__drop_recieved_fds: __drop_recieved_fds:
TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL, 2); TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL, __nr_fds);
if (__fds != NULL) evutil_closesocket(__fds[0]); for (unsigned int i = 0; i < __nr_fds; i++)
if (__fds != NULL) evutil_closesocket(__fds[1]); {
evutil_closesocket(__fds[i]);
}
assert(__nr_fds <= 2);
} }
void * acceptor_kni_v2_event_thread_entry(void * args) void * acceptor_kni_v2_event_thread_entry(void * args)

168
platform/src/ssl_stream.cpp
View File

@@ -76,12 +76,12 @@ enum ssl_stream_stat
SSL_UP_CACHE_SZ, SSL_UP_CACHE_SZ,
SSL_UP_CACHE_QUERY, SSL_UP_CACHE_QUERY,
SSL_UP_CACHE_HIT, SSL_UP_CACHE_HIT,
SSL_DOWN_NEW, SSL_DOWN_NEW,
SSL_DOWN_ERR, SSL_DOWN_ERR,
SSL_DOWN_ERR_NO_CERT, SSL_DOWN_ERR_NO_CERT,
SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK, SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK,
SSL_DOWN_CLOSING, SSL_DOWN_CLOSING,
SSL_DOWN_CLOSED, SSL_DOWN_CLOSED,
SSL_DOWN_DIRTY_CLOSED, SSL_DOWN_DIRTY_CLOSED,
@@ -92,7 +92,7 @@ enum ssl_stream_stat
SSL_DOWN_TICKET_REUSE, SSL_DOWN_TICKET_REUSE,
SSL_DOWN_TICKET_NOTFOUND, SSL_DOWN_TICKET_NOTFOUND,
SSL_DOWN_TIKCET_QUERY, SSL_DOWN_TIKCET_QUERY,
SSL_NO_CHELLO, SSL_NO_CHELLO,
SSL_NO_SNI, SSL_NO_SNI,
SSL_FAKE_CRT, SSL_FAKE_CRT,
@@ -140,7 +140,7 @@ struct ssl_mgr
struct sess_cache * down_sess_cache; struct sess_cache * down_sess_cache;
struct sess_cache * up_sess_cache; struct sess_cache * up_sess_cache;
struct sess_ticket_box * down_stek_box; struct sess_ticket_box * down_stek_box;
struct ssl_service_cache* svc_cache; struct ssl_service_cache* svc_cache;
ssl_stream_new_hook* on_new_upstream_cb; ssl_stream_new_hook* on_new_upstream_cb;
void* upstream_cb_param; void* upstream_cb_param;
@@ -165,7 +165,7 @@ struct ssl_mgr
unsigned int log_master_key; unsigned int log_master_key;
char master_key_file[TFE_PATH_MAX]; char master_key_file[TFE_PATH_MAX];
FILE* fp_master_key; FILE* fp_master_key;
void * logger; void * logger;
screen_stat_handle_t fs_handle; screen_stat_handle_t fs_handle;
long long stat_val[SSL_STAT_MAX]; long long stat_val[SSL_STAT_MAX];
@@ -185,7 +185,7 @@ struct ssl_bypass_condition
}; };
struct ssl_upstream_parts struct ssl_upstream_parts
{ {
struct cert_verify_param verify_param; struct cert_verify_param verify_param;
struct cert_verify_result verify_result; struct cert_verify_result verify_result;
char block_fake_cert; char block_fake_cert;
@@ -194,7 +194,7 @@ struct ssl_upstream_parts
int apln_enabled; int apln_enabled;
int keyring_id; int keyring_id;
struct ssl_chello * client_hello; struct ssl_chello * client_hello;
int is_server_cert_verify_passed; int is_server_cert_verify_passed;
}; };
struct ssl_downstream_parts struct ssl_downstream_parts
{ {
@@ -243,9 +243,9 @@ struct ssl_connect_server_ctx
evutil_socket_t fd_upstream; evutil_socket_t fd_upstream;
evutil_socket_t fd_downstream; evutil_socket_t fd_downstream;
struct tfe_stream* tcp_stream; struct tfe_stream* tcp_stream;
struct future * f_peek_chello; struct future * f_peek_chello;
struct timespec start,end; struct timespec start,end;
}; };
@@ -343,7 +343,7 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
struct ssl_mgr *mgr=(struct ssl_mgr *)arg; struct ssl_mgr *mgr=(struct ssl_mgr *)arg;
int i=0; int i=0;
if(!mgr->no_sesscache) if(!mgr->no_sesscache)
{ {
ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT])); ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT]));
ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT])); ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT]));
} }
@@ -352,14 +352,14 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times; mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times;
mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue; mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue;
mgr->stat_val[KEY_KEEPER_CACHE_SIZE]=keeper_stat.cached_num; mgr->stat_val[KEY_KEEPER_CACHE_SIZE]=keeper_stat.cached_num;
struct ssl_service_cache_statistics svc_stat; struct ssl_service_cache_statistics svc_stat;
memset(&svc_stat, 0, sizeof(svc_stat)); memset(&svc_stat, 0, sizeof(svc_stat));
ssl_service_cache_stat(mgr->svc_cache, &svc_stat); ssl_service_cache_stat(mgr->svc_cache, &svc_stat);
mgr->stat_val[SSL_SVC_PINNING]=svc_stat.pinning_cli_cnt; mgr->stat_val[SSL_SVC_PINNING]=svc_stat.pinning_cli_cnt;
mgr->stat_val[SSL_SVC_MAUTH]=svc_stat.mutual_auth_cli_cnt; mgr->stat_val[SSL_SVC_MAUTH]=svc_stat.mutual_auth_cli_cnt;
mgr->stat_val[SSL_SVC_CT_CERT]=svc_stat.ct_srv_cnt; mgr->stat_val[SSL_SVC_CT_CERT]=svc_stat.ct_srv_cnt;
mgr->stat_val[SSL_SVC_EV_CERT]=svc_stat.ev_srv_cnt; mgr->stat_val[SSL_SVC_EV_CERT]=svc_stat.ev_srv_cnt;
mgr->stat_val[SSL_SVC_APP_NOT_PINNING]=svc_stat.app_not_pinning_cnt; mgr->stat_val[SSL_SVC_APP_NOT_PINNING]=svc_stat.app_not_pinning_cnt;
for(i=0;i<SSL_STAT_MAX;i++) for(i=0;i<SSL_STAT_MAX;i++)
{ {
@@ -379,25 +379,25 @@ void ssl_stat_init(struct ssl_mgr * mgr)
spec[SSL_UP_ERR]="ussl_err"; spec[SSL_UP_ERR]="ussl_err";
spec[SSL_UP_ERR_NO_CIPHER]="ussl_e_ciph"; spec[SSL_UP_ERR_NO_CIPHER]="ussl_e_ciph";
spec[SSL_UP_ERR_UNSUPPORT_PROTO]="ussl_e_prt"; spec[SSL_UP_ERR_UNSUPPORT_PROTO]="ussl_e_prt";
spec[SSL_UP_CLOSING]="ussl_clsing"; spec[SSL_UP_CLOSING]="ussl_clsing";
spec[SSL_UP_CLOSED]="ussl_clsd"; spec[SSL_UP_CLOSED]="ussl_clsd";
spec[SSL_UP_DIRTY_CLOSED]="ussl_dt_cls"; spec[SSL_UP_DIRTY_CLOSED]="ussl_dt_cls";
spec[SSL_UP_CACHE_SZ]="usess_cache"; spec[SSL_UP_CACHE_SZ]="usess_cache";
spec[SSL_UP_CACHE_QUERY]="usess_query"; spec[SSL_UP_CACHE_QUERY]="usess_query";
spec[SSL_UP_CACHE_HIT]="usess_hitn"; spec[SSL_UP_CACHE_HIT]="usess_hitn";
spec[SSL_DOWN_NEW]="dssl_new"; spec[SSL_DOWN_NEW]="dssl_new";
spec[SSL_DOWN_ERR]="dssl_err"; spec[SSL_DOWN_ERR]="dssl_err";
spec[SSL_DOWN_ERR_NO_CERT]="dssl_e_cert"; spec[SSL_DOWN_ERR_NO_CERT]="dssl_e_cert";
spec[SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK]="dssl_e_fb"; spec[SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK]="dssl_e_fb";
spec[SSL_DOWN_CLOSING]="dssl_clsing"; spec[SSL_DOWN_CLOSING]="dssl_clsing";
spec[SSL_DOWN_CLOSED]="dssl_clsd"; spec[SSL_DOWN_CLOSED]="dssl_clsd";
spec[SSL_DOWN_DIRTY_CLOSED]="dssl_dt_cls"; spec[SSL_DOWN_DIRTY_CLOSED]="dssl_dt_cls";
spec[SSL_DOWN_CACHE_SZ]="dsess_cache"; spec[SSL_DOWN_CACHE_SZ]="dsess_cache";
spec[SSL_DOWN_CACHE_QUERY]="dcache_query"; spec[SSL_DOWN_CACHE_QUERY]="dcache_query";
spec[SSL_DOWN_CACHE_HIT]="dsess_hitn"; spec[SSL_DOWN_CACHE_HIT]="dsess_hitn";
if(!mgr->no_sessticket) if(!mgr->no_sessticket)
{ {
spec[SSL_DOWN_TICKET_NEW]="dtkt_new"; spec[SSL_DOWN_TICKET_NEW]="dtkt_new";
@@ -417,7 +417,7 @@ void ssl_stat_init(struct ssl_mgr * mgr)
spec[SSL_SVC_CT_CERT]="ssl_ct_crt"; spec[SSL_SVC_CT_CERT]="ssl_ct_crt";
spec[SSL_SVC_EV_CERT]="ssl_ev_crt"; spec[SSL_SVC_EV_CERT]="ssl_ev_crt";
spec[SSL_SVC_APP_NOT_PINNING]="app_no_pinning"; spec[SSL_SVC_APP_NOT_PINNING]="app_no_pinning";
for(i=0;i<SSL_STAT_MAX;i++) for(i=0;i<SSL_STAT_MAX;i++)
{ {
@@ -431,10 +431,10 @@ void ssl_stat_init(struct ssl_mgr * mgr)
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
value=mgr->fs_id[SSL_UP_CACHE_QUERY]; value=mgr->fs_id[SSL_UP_CACHE_QUERY];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(mgr->fs_handle, FS_register_ratio(mgr->fs_handle,
mgr->fs_id[SSL_UP_CACHE_HIT], mgr->fs_id[SSL_UP_CACHE_HIT],
mgr->fs_id[SSL_UP_CACHE_QUERY], mgr->fs_id[SSL_UP_CACHE_QUERY],
1, 1,
FS_STYLE_STATUS, FS_STYLE_STATUS,
FS_CALC_CURRENT, FS_CALC_CURRENT,
@@ -444,10 +444,10 @@ void ssl_stat_init(struct ssl_mgr * mgr)
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
value=mgr->fs_id[SSL_DOWN_CACHE_QUERY]; value=mgr->fs_id[SSL_DOWN_CACHE_QUERY];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(mgr->fs_handle, FS_register_ratio(mgr->fs_handle,
mgr->fs_id[SSL_DOWN_CACHE_HIT], mgr->fs_id[SSL_DOWN_CACHE_HIT],
mgr->fs_id[SSL_DOWN_CACHE_QUERY], mgr->fs_id[SSL_DOWN_CACHE_QUERY],
1, 1,
FS_STYLE_STATUS, FS_STYLE_STATUS,
FS_CALC_CURRENT, FS_CALC_CURRENT,
@@ -457,21 +457,21 @@ void ssl_stat_init(struct ssl_mgr * mgr)
{ {
value=mgr->fs_id[SSL_DOWN_TIKCET_QUERY]; value=mgr->fs_id[SSL_DOWN_TIKCET_QUERY];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
value=mgr->fs_id[SSL_DOWN_TICKET_REUSE]; value=mgr->fs_id[SSL_DOWN_TICKET_REUSE];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value)); FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(mgr->fs_handle, FS_register_ratio(mgr->fs_handle,
mgr->fs_id[SSL_DOWN_TICKET_REUSE], mgr->fs_id[SSL_DOWN_TICKET_REUSE],
mgr->fs_id[SSL_DOWN_TIKCET_QUERY], mgr->fs_id[SSL_DOWN_TIKCET_QUERY],
1, 1,
FS_STYLE_STATUS, FS_STYLE_STATUS,
FS_CALC_CURRENT, FS_CALC_CURRENT,
"dtkt_hit"); "dtkt_hit");
} }
struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here. struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here.
mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr); mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr);
evtimer_add(mgr->gcev, &gc_delay); evtimer_add(mgr->gcev, &gc_delay);
return; return;
@@ -537,12 +537,12 @@ static void ssl_stream_free(struct ssl_stream * s_stream)
{ {
key_keeper_free_keyring(s_stream->down_parts.keyring); key_keeper_free_keyring(s_stream->down_parts.keyring);
s_stream->down_parts.keyring = NULL; s_stream->down_parts.keyring = NULL;
} }
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED])); ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED]));
break; break;
case CONN_DIR_UPSTREAM: case CONN_DIR_UPSTREAM:
if (s_stream->up_parts.client_hello != NULL) if (s_stream->up_parts.client_hello != NULL)
{ {
ssl_chello_free(s_stream->up_parts.client_hello); ssl_chello_free(s_stream->up_parts.client_hello);
s_stream->up_parts.client_hello = NULL; s_stream->up_parts.client_hello = NULL;
} }
@@ -609,7 +609,7 @@ void ssl_manager_destroy(struct ssl_mgr * mgr)
} }
struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section,
struct event_base * ev_base_gc, void * logger) struct event_base * ev_base_gc, void * logger)
{ {
unsigned int stek_group_num = 0; unsigned int stek_group_num = 0;
@@ -656,7 +656,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
MESA_load_profile_uint_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0); MESA_load_profile_uint_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0);
MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers, MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers,
sizeof(mgr->default_ciphers), DFLT_CIPHERS); sizeof(mgr->default_ciphers), DFLT_CIPHERS);
MESA_load_profile_uint_def(ini_profile, section, "no_session_cache", &(mgr->no_sesscache), 0); MESA_load_profile_uint_def(ini_profile, section, "no_session_cache", &(mgr->no_sesscache), 0);
@@ -670,14 +670,14 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
&(mgr->sess_cache_slots), 4 * 1024 * 1024); &(mgr->sess_cache_slots), 4 * 1024 * 1024);
MESA_load_profile_uint_def(ini_profile, section, "session_cache_expire_seconds", MESA_load_profile_uint_def(ini_profile, section, "session_cache_expire_seconds",
&(mgr->sess_expire_seconds), 30 * 60); &(mgr->sess_expire_seconds), 30 * 60);
if(!mgr->no_sesscache) if(!mgr->no_sesscache)
{ {
mgr->up_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM); mgr->up_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM);
mgr->down_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM); mgr->down_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM);
} }
MESA_load_profile_uint_def(ini_profile, section, "stek_group_num", &stek_group_num, 1); MESA_load_profile_uint_def(ini_profile, section, "stek_group_num", &stek_group_num, 1);
MESA_load_profile_uint_def(ini_profile, section, "stek_rotation_time", &stek_rotation_time, 3600); MESA_load_profile_uint_def(ini_profile, section, "stek_rotation_time", &stek_rotation_time, 3600);
@@ -712,7 +712,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
TFE_LOG_ERROR(logger, "Certificate Manager initiate failed."); TFE_LOG_ERROR(logger, "Certificate Manager initiate failed.");
goto error_out; goto error_out;
} }
MESA_load_profile_uint_def(ini_profile, section, "trusted_cert_load_local", MESA_load_profile_uint_def(ini_profile, section, "trusted_cert_load_local",
&(mgr->trusted_cert_load_local), 1); &(mgr->trusted_cert_load_local), 1);
@@ -721,10 +721,10 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
{ {
MESA_load_profile_string_def(ini_profile, section, "trusted_cert_file", mgr->trusted_cert_file, sizeof(mgr->trusted_cert_file), MESA_load_profile_string_def(ini_profile, section, "trusted_cert_file", mgr->trusted_cert_file, sizeof(mgr->trusted_cert_file),
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"); "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem");
MESA_load_profile_string_def(ini_profile, section, "trusted_cert_dir", mgr->trusted_cert_dir, sizeof(mgr->trusted_cert_dir), MESA_load_profile_string_def(ini_profile, section, "trusted_cert_dir", mgr->trusted_cert_dir, sizeof(mgr->trusted_cert_dir),
"./resource/tfe/trusted_storage"); "./resource/tfe/trusted_storage");
} }
mgr->trust_CA_store = ssl_trusted_cert_storage_create(mgr->trusted_cert_file, mgr->trusted_cert_dir, &(mgr->cert_verify_param)); mgr->trust_CA_store = ssl_trusted_cert_storage_create(mgr->trusted_cert_file, mgr->trusted_cert_dir, &(mgr->cert_verify_param));
if (mgr->trust_CA_store == NULL) if (mgr->trust_CA_store == NULL)
{ {
@@ -746,7 +746,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
mgr->log_master_key=0; mgr->log_master_key=0;
} }
} }
mgr->fs_handle=tfe_proxy_get_fs_handle(); mgr->fs_handle=tfe_proxy_get_fs_handle();
ssl_stat_init(mgr); ssl_stat_init(mgr);
return mgr; return mgr;
@@ -786,7 +786,7 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg)
//use promise_get_ctx instead of promise_dettach_ctx for try more times. //use promise_get_ctx instead of promise_dettach_ctx for try more times.
struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *) promise_get_ctx(promise); struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *) promise_get_ctx(promise);
enum chello_parse_result chello_status=CHELLO_PARSE_INVALID_FORMAT; enum chello_parse_result chello_status=CHELLO_PARSE_INVALID_FORMAT;
struct ssl_chello* chello=NULL; struct ssl_chello* chello=NULL;
const char * reason = NULL; const char * reason = NULL;
unsigned char buf[2048]; unsigned char buf[2048];
ssize_t n = 0; ssize_t n = 0;
@@ -826,7 +826,7 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg)
reason = "too many peek retries"; reason = "too many peek retries";
goto failed; goto failed;
} }
/* ssl_tls_clienthello_parse indicates that we /* ssl_tls_clienthello_parse indicates that we
* should retry later when we have more data, and we * should retry later when we have more data, and we
* haven't reached the maximum retry count yet. * haven't reached the maximum retry count yet.
@@ -836,7 +836,7 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg)
* never actually read them, fd is still ready for * never actually read them, fd is still ready for
* reading now. We use 25 * 0.2 s = 5 s timeout. */ * reading now. We use 25 * 0.2 s = 5 s timeout. */
struct timeval retry_delay = {0, 100}; struct timeval retry_delay = {0, 100};
event_del(ctx->ev); event_del(ctx->ev);
event_free(ctx->ev); event_free(ctx->ev);
ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise); ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise);
assert(ctx->ev != NULL); assert(ctx->ev != NULL);
@@ -942,10 +942,10 @@ static int upstream_ossl_init(struct ssl_stream* s_stream)
TFE_LOG_ERROR(mgr->logger, "ssl set max proto version failed %d.", s_stream->ssl_max_version); TFE_LOG_ERROR(mgr->logger, "ssl set max proto version failed %d.", s_stream->ssl_max_version);
return -1; return -1;
} }
SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL); SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL);
SSL_CTX_set_client_cert_cb(sslctx, ossl_client_cert_cb); SSL_CTX_set_client_cert_cb(sslctx, ossl_client_cert_cb);
if(mgr->no_sesscache) if(mgr->no_sesscache)
{ {
SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_OFF); SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_OFF);
@@ -955,7 +955,7 @@ static int upstream_ossl_init(struct ssl_stream* s_stream)
SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_NO_INTERNAL); SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_NO_INTERNAL);
/* session resuming based on remote endpoint address and port */ /* session resuming based on remote endpoint address and port */
sess = up_session_get(mgr->up_sess_cache, sess = up_session_get(mgr->up_sess_cache,
(struct sockaddr *) &(s_stream->peer_addr), s_stream->peer_addrlen, chello->sni, (struct sockaddr *) &(s_stream->peer_addr), s_stream->peer_addrlen, chello->sni,
s_stream->ssl_min_version, s_stream->ssl_max_version); s_stream->ssl_min_version, s_stream->ssl_max_version);
if (sess) if (sess)
{ {
@@ -1098,7 +1098,7 @@ unsigned long ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir d
} }
if(fs_id>=0) if(fs_id>=0)
{ {
mgr->stat_val[fs_id]++; mgr->stat_val[fs_id]++;
} }
if (!errno && !sslerr) if (!errno && !sslerr)
{ {
@@ -1194,7 +1194,7 @@ void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr
{ {
s_upstream->svc_status.cli_pinning_status=PINNING_ST_PINNING; s_upstream->svc_status.cli_pinning_status=PINNING_ST_PINNING;
ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING); ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING);
ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &s_upstream->svc_status); ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &s_upstream->svc_status);
} }
else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN) else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN)
{ {
@@ -1204,7 +1204,7 @@ void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr
break; break;
case CONN_DIR_UPSTREAM: case CONN_DIR_UPSTREAM:
s_upstream=&(s_stream->up_parts); s_upstream=&(s_stream->up_parts);
s_upstream->svc_status.has_protocol_errors=1; s_upstream->svc_status.has_protocol_errors=1;
ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream->addr, &(s_stream->up_parts.svc_status)); ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream->addr, &(s_stream->up_parts.svc_status));
break; break;
default: default:
@@ -1261,8 +1261,8 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
s_stream->error=SSL_STREAM_R_SERVER_PROTOCOL_ERROR; s_stream->error=SSL_STREAM_R_SERVER_PROTOCOL_ERROR;
} }
else if(events & BEV_EVENT_EOF) else if(events & BEV_EVENT_EOF)
{ {
ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR])); ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
s_stream->error=SSL_STREAM_R_SERVER_CLOSED; s_stream->error=SSL_STREAM_R_SERVER_CLOSED;
} }
else if(events & BEV_EVENT_TIMEOUT) else if(events & BEV_EVENT_TIMEOUT)
@@ -1271,7 +1271,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
s_stream->error=SSL_STREAM_R_CONNECT_SERVER_TIMEOUT; s_stream->error=SSL_STREAM_R_CONNECT_SERVER_TIMEOUT;
} }
else if(events & BEV_EVENT_CONNECTED) else if(events & BEV_EVENT_CONNECTED)
{ {
bufferevent_disable(ctx->bev, EV_READ | EV_WRITE); bufferevent_disable(ctx->bev, EV_READ | EV_WRITE);
bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success
clock_gettime(CLOCK_MONOTONIC, &(ctx->end)); clock_gettime(CLOCK_MONOTONIC, &(ctx->end));
@@ -1279,7 +1279,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS) if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS)
{ {
TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s", TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s",
jiffies_ms, jiffies_ms,
s_stream->tcp_stream->str_stream_info, s_stream->tcp_stream->str_stream_info,
s_upstream->client_hello->sni); s_upstream->client_hello->sni);
} }
@@ -1293,8 +1293,8 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
s_upstream->is_server_cert_verify_passed=1; s_upstream->is_server_cert_verify_passed=1;
} }
else else
{ {
s_upstream->is_server_cert_verify_passed = ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store, s_upstream->is_server_cert_verify_passed = ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store,
s_stream->ssl, s_stream->up_parts.client_hello->sni, &(s_stream->up_parts.verify_param), s_stream->ssl, s_stream->up_parts.client_hello->sni, &(s_stream->up_parts.verify_param),
error_str, sizeof(error_str), &(s_stream->up_parts.verify_result)); error_str, sizeof(error_str), &(s_stream->up_parts.verify_result));
s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct; s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct;
@@ -1333,7 +1333,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
else else
{ {
//Do not perform cert verification on reused session. //Do not perform cert verification on reused session.
s_upstream->is_server_cert_verify_passed=1; s_upstream->is_server_cert_verify_passed=1;
} }
if(mgr->log_master_key) if(mgr->log_master_key)
{ {
@@ -1478,7 +1478,7 @@ void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_s
ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p); ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p);
ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger); ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger);
} }
static int ossl_session_ticket_key_callback(SSL *ssl_conn, static int ossl_session_ticket_key_callback(SSL *ssl_conn,
unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx,
@@ -1494,14 +1494,14 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn,
const char * sni = s_stream->peer->up_parts.client_hello->sni; const char * sni = s_stream->peer->up_parts.client_hello->sni;
struct sess_ticket_box * stek_box = s_stream->mgr->down_stek_box; struct sess_ticket_box * stek_box = s_stream->mgr->down_stek_box;
struct sess_ticket_key cur_stek; struct sess_ticket_key cur_stek;
unsigned char buf[33]={0}; unsigned char buf[33]={0};
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY])); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY]));
if (enc == 1) if (enc == 1)
{ {
sess_ticket_box_get_key_for_enc(stek_box, sni, &cur_stek); sess_ticket_box_get_key_for_enc(stek_box, sni, &cur_stek);
/* encrypt session stek */ /* encrypt session stek */
if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1) if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1)
{ {
TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed"); TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed");
@@ -1514,7 +1514,7 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn,
ret=STEK_ERROR; ret=STEK_ERROR;
goto leave; goto leave;
} }
if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1) if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1)
{ {
TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed"); TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed");
ret=STEK_ERROR; ret=STEK_ERROR;
@@ -1523,10 +1523,10 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn,
memcpy(name, cur_stek.name, sizeof(cur_stek.name)); memcpy(name, cur_stek.name, sizeof(cur_stek.name));
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW])); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW]));
ret=STEK_FOUND_FRESH; ret=STEK_FOUND_FRESH;
} }
else else
{ {
/* decrypt session stek */ /* decrypt session stek */
ret=sess_ticket_box_get_key_for_dec(stek_box, sni, name, &cur_stek); ret=sess_ticket_box_get_key_for_dec(stek_box, sni, name, &cur_stek);
if(ret==STEK_FOUND_FRESH||ret==STEK_FOUND_STALED) if(ret==STEK_FOUND_FRESH||ret==STEK_FOUND_STALED)
@@ -1536,8 +1536,8 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn,
TFE_LOG_ERROR(logger, "HMAC_Init_ex() failed"); TFE_LOG_ERROR(logger, "HMAC_Init_ex() failed");
ret= STEK_ERROR; ret= STEK_ERROR;
goto leave; goto leave;
} }
if (EVP_DecryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1) if (EVP_DecryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1)
{ {
TFE_LOG_ERROR(logger, "EVP_DecryptInit_ex() failed"); TFE_LOG_ERROR(logger, "EVP_DecryptInit_ex() failed");
ret= STEK_ERROR; ret= STEK_ERROR;
@@ -1552,7 +1552,7 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn,
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND])); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND]));
} }
} }
leave: leave:
return (int)ret; return (int)ret;
} }
@@ -1639,7 +1639,7 @@ static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr)
{ {
SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2); SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2);
} }
if (mgr->no_ssl3) if (mgr->no_ssl3)
{ {
SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3); SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3);
} }
@@ -1780,7 +1780,7 @@ void downstream_ossl_init(struct ssl_stream *s_stream)
ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream); ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream);
assert(ret == 1); assert(ret == 1);
if (mgr->ssl_mode_release_buffers == 1) if (mgr->ssl_mode_release_buffers == 1)
{ {
/* lower memory footprint for idle connections */ /* lower memory footprint for idle connections */
@@ -1794,7 +1794,7 @@ void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx)
{ {
X509_free(ctx->origin_crt); X509_free(ctx->origin_crt);
ctx->origin_crt=NULL; ctx->origin_crt=NULL;
if (ctx->f_ask_keyring != NULL) if (ctx->f_ask_keyring != NULL)
{ {
future_destroy(ctx->f_ask_keyring); future_destroy(ctx->f_ask_keyring);
@@ -1842,9 +1842,9 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events,
struct ssl_stream * s_stream = ctx->downstream; struct ssl_stream * s_stream = ctx->downstream;
struct ssl_upstream_parts* s_upstream= &(ctx->peer->up_parts); struct ssl_upstream_parts* s_upstream= &(ctx->peer->up_parts);
struct ssl_mgr* mgr=s_stream->mgr; struct ssl_mgr* mgr=s_stream->mgr;
char error_str[TFE_STRING_MAX]={0}; char error_str[TFE_STRING_MAX]={0};
uint64_t jiffies_ms=0; uint64_t jiffies_ms=0;
unsigned long sslerr=0; unsigned long sslerr=0;
if (events & BEV_EVENT_ERROR) if (events & BEV_EVENT_ERROR)
{ {
@@ -1857,7 +1857,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events,
s_stream->error=SSL_STREAM_R_CLIENT_PROTOCOL_ERROR; s_stream->error=SSL_STREAM_R_CLIENT_PROTOCOL_ERROR;
} }
else if(events & BEV_EVENT_EOF) else if(events & BEV_EVENT_EOF)
{ {
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR])); ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
ssl_stream_process_zero_eof(s_stream, mgr); ssl_stream_process_zero_eof(s_stream, mgr);
} }
@@ -1868,14 +1868,14 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events,
} }
else if(events & BEV_EVENT_CONNECTED) else if(events & BEV_EVENT_CONNECTED)
{ {
clock_gettime(CLOCK_MONOTONIC, &(ctx->end)); clock_gettime(CLOCK_MONOTONIC, &(ctx->end));
jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000; jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000;
if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS) if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS)
{ {
TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s", TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s",
jiffies_ms, jiffies_ms,
s_stream->tcp_stream->str_stream_info, s_stream->tcp_stream->str_stream_info,
s_upstream->client_hello->sni); s_upstream->client_hello->sni);
} }
s_stream->connect_latency_ms=jiffies_ms; s_stream->connect_latency_ms=jiffies_ms;
@@ -1895,7 +1895,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events,
// ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &svc_status); // ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &svc_status);
promise_success(p, ctx); promise_success(p, ctx);
} }
if(s_stream->error) if(s_stream->error)
{ {
ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error)); ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error));
@@ -1917,7 +1917,7 @@ void ask_keyring_on_succ(void * result, void * user)
struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id); struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id);
kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing. kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing.
clock_gettime(CLOCK_MONOTONIC, &(ctx->start)); clock_gettime(CLOCK_MONOTONIC, &(ctx->start));
ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL,
kyr, ctx->peer, ctx->tcp_stream); kyr, ctx->peer, ctx->tcp_stream);
@@ -1925,7 +1925,7 @@ void ask_keyring_on_succ(void * result, void * user)
ctx->bev_down = bufferevent_openssl_socket_new(evbase, ctx->fd_downstream, ctx->downstream->ssl, ctx->bev_down = bufferevent_openssl_socket_new(evbase, ctx->fd_downstream, ctx->downstream->ssl,
BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE); BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE);
bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1); bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1);
bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p); bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p);
bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only
@@ -2121,7 +2121,7 @@ complete:
* Guarantees that SSL and the corresponding SSL_CTX are freed and the * Guarantees that SSL and the corresponding SSL_CTX are freed and the
* socket is closed, eventually, or in the case of fatal errors, immediately. * socket is closed, eventually, or in the case of fatal errors, immediately.
*/ */
void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev) void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev)
{ {
UNUSED struct ssl_shutdown_ctx * sslshutctx = NULL; UNUSED struct ssl_shutdown_ctx * sslshutctx = NULL;
evutil_socket_t fd=-1; evutil_socket_t fd=-1;
@@ -2165,8 +2165,8 @@ void ssl_stream_free_and_close_fd(struct ssl_stream * s_stream, struct event_bas
bufferevent_setcb(ubev, NULL, NULL, NULL, NULL); bufferevent_setcb(ubev, NULL, NULL, NULL, NULL);
bufferevent_free(ubev); bufferevent_free(ubev);
} }
ssl_stream_free(s_stream); ssl_stream_free(s_stream);
evutil_closesocket(fd);
} }
int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file) int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file)
@@ -2301,4 +2301,4 @@ int ssl_stream_get_string_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT o
return -1; return -1;
} }
return 0; return 0;
} }

69
platform/src/tcp_stream.cpp
View File

@@ -401,7 +401,11 @@ evutil_socket_t __conn_private_release_fd(struct tfe_conn_private * conn)
static void __conn_private_destory(struct tfe_conn_private * conn) static void __conn_private_destory(struct tfe_conn_private * conn)
{ {
bufferevent_disable(conn->bev, EV_READ | EV_WRITE); if (conn->bev)
{
bufferevent_disable(conn->bev, EV_READ | EV_WRITE);
bufferevent_free(conn->bev);
}
if(conn->ratelimit_bucket) if(conn->ratelimit_bucket)
{ {
@@ -409,10 +413,9 @@ static void __conn_private_destory(struct tfe_conn_private * conn)
conn->ratelimit_bucket = NULL; conn->ratelimit_bucket = NULL;
} }
bufferevent_free(conn->bev);
if (conn->fd > 0) if (conn->fd > 0)
{ {
evutil_closesocket(conn->fd); assert(evutil_closesocket(conn->fd) >= 0);
} }
free(conn); free(conn);
@@ -423,8 +426,7 @@ static void __conn_private_destory_with_ssl(struct event_base * ev_base,
struct tfe_conn_private * conn, struct ssl_stream * ssl_stream) struct tfe_conn_private * conn, struct ssl_stream * ssl_stream)
{ {
if (ssl_stream == NULL) return __conn_private_destory(conn); if (ssl_stream == NULL) return __conn_private_destory(conn);
ssl_stream_free_and_close_fd(ssl_stream, ev_base, conn->bev); ssl_stream_free(ssl_stream, ev_base, conn->bev);
conn->fd = -1;
return __conn_private_destory(conn); return __conn_private_destory(conn);
} }
@@ -950,14 +952,15 @@ void ssl_downstream_create_on_success(future_result_t * result, void * user)
{ {
struct tfe_stream_private * _stream = (struct tfe_stream_private *) user; struct tfe_stream_private * _stream = (struct tfe_stream_private *) user;
struct ssl_stream * downstream = ssl_downstream_create_result_release_stream(result); struct ssl_stream * downstream = ssl_downstream_create_result_release_stream(result);
struct bufferevent * bev = ssl_downstream_create_result_release_bev(result); struct bufferevent * bev = ssl_downstream_create_result_release_bev(result);
_stream->defer_fd_downstream = 0;
_stream->conn_downstream = __conn_private_create_by_bev(_stream, bev); _stream->conn_downstream = __conn_private_create_by_bev(_stream, bev);
_stream->ssl_downstream = downstream; _stream->ssl_downstream = downstream;
future_destroy(_stream->future_downstream_create); future_destroy(_stream->future_downstream_create);
_stream->future_downstream_create = NULL; _stream->future_downstream_create = NULL;
_stream->defer_fd_downstream = 0;
assert(_stream->conn_downstream != NULL && _stream->conn_upstream != NULL); assert(_stream->conn_downstream != NULL && _stream->conn_upstream != NULL);
@@ -988,19 +991,17 @@ void ssl_downstream_create_on_fail(enum e_future_error err, const char * what, v
void ssl_upstream_create_on_success(future_result_t * result, void * user) void ssl_upstream_create_on_success(future_result_t * result, void * user)
{ {
struct tfe_stream_private * _stream = (struct tfe_stream_private *) user; struct tfe_stream_private * _stream = (struct tfe_stream_private *) user;
evutil_socket_t fd=-1; evutil_socket_t fd = -1;
enum ssl_stream_action ssl_action=ssl_upstream_create_result_release_action(result); enum ssl_stream_action ssl_action = ssl_upstream_create_result_release_action(result);
if(SSL_ACTION_PASSTHROUGH==ssl_action) if (SSL_ACTION_PASSTHROUGH == ssl_action)
{ {
_stream->tcp_passthough=1; _stream->tcp_passthough = 1;
fd=ssl_upstream_create_result_release_fd(result); _stream->conn_upstream = __conn_private_create_by_fd(_stream, _stream->defer_fd_upstream);
_stream->conn_upstream=__conn_private_create_by_fd(_stream, fd); _stream->conn_downstream = __conn_private_create_by_fd(_stream, _stream->defer_fd_downstream);
_stream->conn_downstream=__conn_private_create_by_fd(_stream, _stream->defer_fd_downstream);
__conn_private_enable(_stream->conn_downstream); __conn_private_enable(_stream->conn_downstream);
__conn_private_enable(_stream->conn_upstream); __conn_private_enable(_stream->conn_upstream);
} }
else if(SSL_ACTION_SHUTDOWN==ssl_action) else if (SSL_ACTION_SHUTDOWN == ssl_action)
{ {
tfe_stream_destory(_stream); tfe_stream_destory(_stream);
} }
@@ -1010,7 +1011,8 @@ void ssl_upstream_create_on_success(future_result_t * result, void * user)
struct bufferevent * bev = ssl_upstream_create_result_release_bev(result); struct bufferevent * bev = ssl_upstream_create_result_release_bev(result);
assert(upstream != NULL && bev != NULL); assert(upstream != NULL && bev != NULL);
/* Create connection ctx by bev */ /* Create connection ctx by bev, fd's ownership is transfer to bev */
_stream->defer_fd_upstream = 0;
_stream->conn_upstream = __conn_private_create_by_bev(_stream, bev); _stream->conn_upstream = __conn_private_create_by_bev(_stream, bev);
_stream->ssl_upstream = upstream; _stream->ssl_upstream = upstream;
@@ -1024,10 +1026,9 @@ void ssl_upstream_create_on_success(future_result_t * result, void * user)
ssl_async_downstream_create(_stream->future_downstream_create, _stream->ssl_mgr, ssl_async_downstream_create(_stream->future_downstream_create, _stream->ssl_mgr,
_stream->ssl_upstream, _stream->defer_fd_downstream, &_stream->head); _stream->ssl_upstream, _stream->defer_fd_downstream, &_stream->head);
} }
future_destroy(_stream->future_upstream_create); future_destroy(_stream->future_upstream_create);
_stream->future_upstream_create = NULL; _stream->future_upstream_create = NULL;
_stream->defer_fd_upstream = 0;
} }
void ssl_upstream_create_on_fail(enum e_future_error err, const char * what, void * user) void ssl_upstream_create_on_fail(enum e_future_error err, const char * what, void * user)
@@ -1126,21 +1127,23 @@ void tfe_stream_destory(struct tfe_stream_private * stream)
if (__is_ssl(stream) && stream->ssl_upstream) if (__is_ssl(stream) && stream->ssl_upstream)
{ {
ssl_stream_free_and_close_fd(stream->ssl_upstream, ev_base, stream->conn_upstream->bev); ssl_stream_free(stream->ssl_upstream, ev_base, stream->conn_upstream->bev);
} }
if (__is_ssl(stream) && stream->ssl_downstream) if (__is_ssl(stream) && stream->ssl_downstream)
{ {
ssl_stream_free_and_close_fd(stream->ssl_downstream, ev_base, stream->conn_downstream->bev); ssl_stream_free(stream->ssl_downstream, ev_base, stream->conn_downstream->bev);
} }
if (stream->conn_upstream) if (stream->conn_upstream)
{ {
assert(stream->defer_fd_upstream <= 0);
__conn_private_destory(stream->conn_upstream); __conn_private_destory(stream->conn_upstream);
} }
if (stream->conn_downstream) if (stream->conn_downstream)
{ {
assert(stream->defer_fd_downstream <= 0);
__conn_private_destory(stream->conn_downstream); __conn_private_destory(stream->conn_downstream);
} }
@@ -1327,13 +1330,21 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst
if (_stream->session_type == STREAM_PROTO_PLAIN) if (_stream->session_type == STREAM_PROTO_PLAIN)
{ {
_stream->conn_downstream = __conn_private_create_by_fd(_stream, fd_downstream); _stream->conn_downstream = __conn_private_create_by_fd(_stream, fd_downstream);
if (_stream->conn_downstream != NULL)
{
_stream->defer_fd_downstream = 0;
}
else
{
goto __errout;
}
_stream->conn_upstream = __conn_private_create_by_fd(_stream, fd_upstream); _stream->conn_upstream = __conn_private_create_by_fd(_stream, fd_upstream);
if (_stream->conn_upstream != NULL)
/* Defer FD has been transfer to conn_downstream/conn_upstream */ {
_stream->defer_fd_downstream = 0; _stream->defer_fd_downstream = 0;
_stream->defer_fd_upstream = 0; }
else
if (unlikely(_stream->conn_downstream == NULL || _stream->conn_upstream == NULL))
{ {
goto __errout; goto __errout;
} }
@@ -1364,6 +1375,12 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst
return 0; return 0;
__errout: __errout:
/* The fds not been accept by this function, clear up and release at caller */
_stream->defer_fd_downstream = 0;
_stream->defer_fd_upstream = 0;
_stream->log_fd_downstream = 0;
_stream->log_fd_upstream = 0;
return -1; return -1;
} }

3
script/CMakeLists.txt
View File

@@ -1,5 +1,4 @@
install(PROGRAMS user/r2_tfe DESTINATION ./ COMPONENT Program)
install(PROGRAMS user/r3_tfe DESTINATION ./ COMPONENT Program)
install(FILES sysctl/80-tfe.conf DESTINATION /etc/sysctl.d/ COMPONENT Program) install(FILES sysctl/80-tfe.conf DESTINATION /etc/sysctl.d/ COMPONENT Program)
install(FILES service/tfe.service DESTINATION /usr/lib/systemd/system/ COMPONENT Program)
install(FILES service/tfe-env.service DESTINATION /usr/lib/systemd/system/ COMPONENT Program) install(FILES service/tfe-env.service DESTINATION /usr/lib/systemd/system/ COMPONENT Program)
install(FILES service/tfe-env-config DESTINATION /etc/sysconfig/ COMPONENT Profile) install(FILES service/tfe-env-config DESTINATION /etc/sysconfig/ COMPONENT Profile)

6
script/service/tfe-env.service
View File

@@ -1,5 +1,7 @@
[Unit] [Unit]
Description=Tango Frontend Engine - Running Environment Setup Description=Tango Frontend Engine - Running Environment Setup
Before=tfe.service
Wants=network-online.target
[Service] [Service]
EnvironmentFile=/etc/sysconfig/tfe-env-config EnvironmentFile=/etc/sysconfig/tfe-env-config
@@ -10,7 +12,7 @@ RemainAfterExit=yes
ExecStart=/bin/true ExecStart=/bin/true
ExecStop=/bin/true ExecStop=/bin/true
# dataincoming interface ExecStartPost=/usr/sbin/modprobe tfe-kmod
ExecStartPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} address ${TFE_LOCAL_MAC_DATA_INCOMING} ExecStartPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} address ${TFE_LOCAL_MAC_DATA_INCOMING}
ExecStartPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} up ExecStartPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} up
ExecStartPost=/usr/sbin/ip addr flush dev ${TFE_DEVICE_DATA_INCOMING} ExecStartPost=/usr/sbin/ip addr flush dev ${TFE_DEVICE_DATA_INCOMING}
@@ -41,6 +43,8 @@ ExecStopPost=/usr/sbin/ip -6 route del default via fd00::01
ExecStopPost=/usr/sbin/ip -6 route del local default dev lo table 102 ExecStopPost=/usr/sbin/ip -6 route del local default dev lo table 102
ExecStopPost=/usr/sbin/ip addr del fd00::02/64 dev ${TFE_DEVICE_DATA_INCOMING} ExecStopPost=/usr/sbin/ip addr del fd00::02/64 dev ${TFE_DEVICE_DATA_INCOMING}
ExecStopPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} down ExecStopPost=/usr/sbin/ip link set ${TFE_DEVICE_DATA_INCOMING} down
ExecStopPost=/usr/sbin/modprobe -r tfe-kmod
[Install] [Install]
RequiredBy=tfe.service
WantedBy=multi-user.target WantedBy=multi-user.target

20
script/service/tfe.service Normal file
View File

@@ -0,0 +1,20 @@
[Unit]
Description=Tango Frontend Engine
Requires=tfe-env.service
After=tfe-env.service
[Service]
Type=simple
ExecStart=/opt/tsg/tfe/bin/tfe
TimeoutSec=180s
RestartSec=10s
Restart=always
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target