gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
This repository has been archived on 2025-09-14. You can view files and clone it, but cannot push or open issues or pull requests.
Files
b1f670c05da4a83df2ba68db62a0c1a91f614d93
tango-tfe/platform/src/ssl_stream.cpp
luwenpeng c3b887f1c5 perf: 性能优化 
* io_uring使用buffer pool避免内存分配与释放
    * packet io thread与worker thread无锁访问cmsg
    * 为解密流量的fd设置默认的TTL
2023-11-14 18:54:40 +08:00

2279 lines
70 KiB
C++
Raw Blame History

/*
* SSL stream implementation.
* Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise.
* Author: zhengchao@iie.ac.cn
* Create: 2018-8-17
*/
#include <netinet/in.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <stdint.h>
#include <sys/socket.h>
#include <event2/event.h>
#include <event2/listener.h>
#include <event2/bufferevent.h>
#include <event2/bufferevent_ssl.h>
#include <event2/buffer.h>
#include <event2/thread.h>
#include <event2/dns.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <openssl/x509.h>
#include <openssl/x509v3.h>
#include <MESA/MESA_htable.h>
#include <MESA/MESA_prof_load.h>
#include <ssl_fetch_cert.h>
#include <tfe_stream.h>
#include <tfe_utils.h>
#include <tfe_future.h>
#include <tfe_proxy.h>
#include <key_keeper.h>
#include <ssl_sess_cache.h>
#include <ssl_sess_ticket.h>
#include <ssl_trusted_cert_storage.h>
#include <ssl_utils.h>
#include <ssl_service_cache.h>
#include <platform.h>
#include <ssl_ja3.h>
static int SSL_CTX_EX_DATA_IDX_SSLMGR;
static int SSL_EX_DATA_IDX_SSLSTREAM;
static unsigned int ssl_debug;
#define MAX_NET_RETRIES 50
#define LATENCY_WARNING_THRESHOLD_MS 1000
/*
* Default cipher suite spec.
* Use 'openssl ciphers -v spec' to see what ciphers are effectively enabled
* by a cipher suite spec with a given version of OpenSSL.
*/
#define DFLT_CIPHERS "ALL:-aNULL"
static unsigned char SSL_ALPN_HTTP_1_1[]={8, 'h', 't', 't', 'p', '/', '1', '.', '1',0};
static unsigned char SSL_ALPN_HTTP_2[]={2, 'h','2',0};
/*
* Default elliptic curve for EC cipher suites.
*/
#define DFLT_CURVE "prime256v1"
extern long long certstore_is_unavailable;
enum ssl_stream_stat
{
SSL_UP_NEW,
SSL_UP_ERR,
SSL_UP_ERR_NO_CIPHER,
SSL_UP_ERR_UNSUPPORT_PROTO,
SSL_UP_CLOSING,
SSL_UP_CLOSED,
SSL_UP_DIRTY_CLOSED,
SSL_UP_CACHE_SZ,
SSL_UP_CACHE_QUERY,
SSL_UP_CACHE_HIT,
SSL_DOWN_NEW,
SSL_DOWN_ERR,
SSL_DOWN_ERR_NO_CERT,
SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK,
SSL_DOWN_CLOSING,
SSL_DOWN_CLOSED,
SSL_DOWN_DIRTY_CLOSED,
SSL_DOWN_CACHE_SZ,
SSL_DOWN_CACHE_QUERY,
SSL_DOWN_CACHE_HIT,
SSL_DOWN_TICKET_NEW,
SSL_DOWN_TICKET_REUSE,
SSL_DOWN_TICKET_NOTFOUND,
SSL_DOWN_TIKCET_QUERY,
SSL_NO_CHELLO,
SSL_NO_SNI,
SSL_FAKE_CRT,
KEY_KEEPER_CACHE_SIZE,
KEY_KEEPER_ASK,
KEY_KEEPER_ISSUE,
SSL_SVC_PINNING,
SSL_SVC_MAUTH,
SSL_SVC_CT_CERT,
SSL_SVC_EV_CERT,
SSL_SVC_APP_NOT_PINNING,
SSL_SVC_TRUSTED_CERT,
SSL_STAT_MAX
};
struct ssl_mgr
{
unsigned int sslcomp;
unsigned int no_ssl2;
unsigned int no_ssl3;
unsigned int no_tls10;
unsigned int no_tls11;
unsigned int no_tls12;
unsigned int no_sesscache;
unsigned int no_sessticket;
unsigned int no_alpn;
unsigned int no_cert_verify;
unsigned int no_mirror_client_cipher_suite;
CONST_SSL_METHOD * (* sslmethod)(void); //Parameter of SSL_CTX_new
int ssl_min_version, ssl_max_version;
char ssl_session_context[8];
unsigned int sess_cache_slots;
unsigned int sess_expire_seconds;
unsigned int svc_cache_slots;
unsigned int svc_expire_seconds;
unsigned int svc_fail_as_pinning_cnt;
unsigned int svc_fail_as_proto_err_cnt;
unsigned int svc_succ_as_app_not_pinning_cnt;
unsigned int svc_cnt_time_window;
struct sess_cache * down_sess_cache;
struct sess_cache * up_sess_cache;
struct sess_ticket_box * down_stek_box;
struct ssl_service_cache* svc_cache;
ssl_stream_new_hook* on_new_upstream_cb;
void* upstream_cb_param;
struct sess_ticket_key ticket_key;
char default_ciphers[TFE_SYMBOL_MAX];
DH * dh;
char * ecdhcurve;
char * crl_url;
unsigned int trusted_cert_load_local;
struct cert_store_param cert_verify_param;
uint8_t ssl_mode_release_buffers;
char trusted_cert_file[TFE_PATH_MAX];
char trusted_cert_dir[TFE_PATH_MAX];
struct ssl_trusted_cert_storage * trust_CA_store;
struct key_keeper * key_keeper;
struct event_base * ev_base_gc;
struct event * gcev;
unsigned int log_master_key;
char master_key_file[TFE_PATH_MAX];
FILE* fp_master_key;
void * logger;
screen_stat_handle_t fs_handle;
long long stat_val[SSL_STAT_MAX];
int fs_id[SSL_STAT_MAX];
};
struct __ssl_stream_debug
{
evutil_socket_t fd;
};
struct ssl_bypass_condition
{
char bypass_ev_cert;
char bypass_ct_cert;
char bypass_mutual_auth;
char bypass_cert_pinning;
};
struct ssl_upstream_parts
{
struct cert_verify_param verify_param;
struct cert_verify_result verify_result;
char block_fake_cert;
struct ssl_service_status svc_status;
enum ssl_stream_action action;
int apln_enabled;
int keyring_for_trusted;
int keyring_for_untrusted;
struct ssl_chello * client_hello;
uint8_t is_server_cert_verify_passed;
};
struct ssl_downstream_parts
{
struct keyring * keyring;
};
struct ssl_stream
{
enum tfe_conn_dir dir;
SSL * ssl;
struct ssl_mgr * mgr;
union
{
struct ssl_upstream_parts up_parts;
struct ssl_downstream_parts down_parts;
};
int ssl_min_version, ssl_max_version, negotiated_version;
const unsigned char* alpn_selected; //reference to SSL_ALPN_HTTP_2/SSL_ALPN_HTTP_1_1
struct tfe_stream* tcp_stream;
uint64_t connect_latency_ms;
struct ssl_stream* peer;
socklen_t peer_addrlen;
struct sockaddr_storage peer_addr;
struct __ssl_stream_debug _do_not_use;
enum ssl_stream_error error;
};
struct peek_client_hello_ctx
{
struct ssl_chello* chello;
unsigned char sni_peek_retries; /* max 64 SNI parse retries */
struct event * ev;
struct event_base * evbase;
void * logger;
};
struct ssl_connect_server_ctx
{
struct bufferevent * bev;
struct ssl_stream * s_stream;
struct ssl_mgr * mgr;
struct sockaddr_storage addr;
socklen_t addrlen;
void * logger;
evutil_socket_t fd_upstream;
evutil_socket_t fd_downstream;
struct tfe_stream* tcp_stream;
struct future * f_peek_chello;
struct timespec start,end;
};
struct ssl_connect_client_ctx
{
struct tfe_stream* tcp_stream;
struct ssl_stream * peer;
X509 * origin_crt;
int is_origin_crt_verify_passed;
struct ssl_mgr * ssl_mgr;
evutil_socket_t fd_downstream;
struct future * f_ask_keyring;
struct bufferevent * bev_down;
struct ssl_stream * downstream;
struct timespec start,end;
};
/*
* SSL shutdown context.
*/
struct ssl_shutdown_ctx
{
struct ssl_stream * s_stream;
struct event_base * evbase;
struct event * ev;
struct ssl_mgr* mgr;
enum tfe_conn_dir dir;
unsigned int retries;
};
struct fs_spec
{
enum ssl_stream_stat id;
const char* name;
};
unsigned int is_ssl_debug()
{
return ssl_debug;
}
int sslver_str2num(const char * version_str)
{
int sslversion = -1;
assert(OPENSSL_VERSION_NUMBER >= 0x10100000L);
/*
* Support for SSLv2 and the corresponding SSLv2_method(),
* SSLv2_server_method() and SSLv2_client_method() functions were
* removed in OpenSSL 1.1.0.
*/
if (!strcasecmp(version_str, "ssl3")||!strcasecmp(version_str, "SSLv3")||!strcasecmp(version_str, "sslv3.0"))
{
sslversion = SSL3_VERSION;
}
else if (!strcasecmp(version_str, "tls10") || !strcasecmp(version_str, "tls1") ||
!strcasecmp(version_str, "TLSv1.0") || !strcasecmp(version_str, "TLSv1"))
{
sslversion = TLS1_VERSION;
}
else if (!strcasecmp(version_str, "tls11")||!strcasecmp(version_str, "TLSv1.1"))
{
sslversion = TLS1_1_VERSION;
}
else if (!strcasecmp(version_str, "tls12")||!strcasecmp(version_str, "TLSv1.2"))
{
sslversion = TLS1_2_VERSION;
}
else if (!strcasecmp(version_str, "tls13")||!strcasecmp(version_str, "TLSv1.3"))
{
sslversion = TLS1_3_VERSION;
}
else
{
sslversion = -1;
}
return sslversion;
}
const char* ssl_stream_get_error_string(enum ssl_stream_error error)
{
const char* ssl_err_str[__SSL_STREAM_R_MAX];
ssl_err_str[SSL_STREAM_R_NO_ERROR]="OK";
ssl_err_str[SSL_STREAM_R_SERVER_CLOSED]="server-side closed";
ssl_err_str[SSL_STREAM_R_CLIENT_CLOSED]="client-side closed";
ssl_err_str[SSL_STREAM_R_CONNECT_SERVER_TIMEOUT]="server-side timeout";
ssl_err_str[SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT]="client-side timeout";
ssl_err_str[SSL_STREAM_R_SERVER_PROTOCOL_ERROR]="server-side protocol errors";
ssl_err_str[SSL_STREAM_R_CLIENT_PROTOCOL_ERROR]="client-side protocol errors";
return ssl_err_str[error];
}
/*
* Garbage collection handler.
*/
static void
ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
{
struct ssl_mgr *mgr=(struct ssl_mgr *)arg;
int i=0;
if(!mgr->no_sesscache)
{
ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT]));
ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT]));
}
struct key_keeper_stat keeper_stat = { 0 };
key_keeper_statistic(mgr->key_keeper, &keeper_stat);
mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times;
mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue;
mgr->stat_val[KEY_KEEPER_CACHE_SIZE]=keeper_stat.cached_num;
struct ssl_service_cache_statistics svc_stat;
memset(&svc_stat, 0, sizeof(svc_stat));
ssl_service_cache_stat(mgr->svc_cache, &svc_stat);
mgr->stat_val[SSL_SVC_PINNING]=svc_stat.pinning_cli_cnt;
mgr->stat_val[SSL_SVC_MAUTH]=svc_stat.mutual_auth_cli_cnt;
mgr->stat_val[SSL_SVC_CT_CERT]=svc_stat.ct_srv_cnt;
mgr->stat_val[SSL_SVC_EV_CERT]=svc_stat.ev_srv_cnt;
mgr->stat_val[SSL_SVC_APP_NOT_PINNING]=svc_stat.app_not_pinning_cnt;
mgr->stat_val[SSL_SVC_TRUSTED_CERT]=svc_stat.trusted_cert_cnt;
for(i=0;i<SSL_STAT_MAX;i++)
{
FS_operate(mgr->fs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i])));
}
if(mgr->log_master_key && mgr->fp_master_key)
{
fflush(mgr->fp_master_key);
}
return;
}
void ssl_stat_init(struct ssl_mgr * mgr)
{
int i=0;
const char* spec[SSL_STAT_MAX]={0};
spec[SSL_UP_NEW]="ussl_new";
spec[SSL_UP_ERR]="ussl_err";
spec[SSL_UP_ERR_NO_CIPHER]="ussl_e_ciph";
spec[SSL_UP_ERR_UNSUPPORT_PROTO]="ussl_e_prt";
spec[SSL_UP_CLOSING]="ussl_clsing";
spec[SSL_UP_CLOSED]="ussl_clsd";
spec[SSL_UP_DIRTY_CLOSED]="ussl_dt_cls";
spec[SSL_UP_CACHE_SZ]="usess_cache";
spec[SSL_UP_CACHE_QUERY]="usess_query";
spec[SSL_UP_CACHE_HIT]="usess_hitn";
spec[SSL_DOWN_NEW]="dssl_new";
spec[SSL_DOWN_ERR]="dssl_err";
spec[SSL_DOWN_ERR_NO_CERT]="dssl_e_cert";
spec[SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK]="dssl_e_fb";
spec[SSL_DOWN_CLOSING]="dssl_clsing";
spec[SSL_DOWN_CLOSED]="dssl_clsd";
spec[SSL_DOWN_DIRTY_CLOSED]="dssl_dt_cls";
spec[SSL_DOWN_CACHE_SZ]="dsess_cache";
spec[SSL_DOWN_CACHE_QUERY]="dcache_query";
spec[SSL_DOWN_CACHE_HIT]="dsess_hitn";
if(!mgr->no_sessticket)
{
spec[SSL_DOWN_TICKET_NEW]="dtkt_new";
spec[SSL_DOWN_TICKET_REUSE]="dtkt_reuse";
spec[SSL_DOWN_TICKET_NOTFOUND]="dtkt_notfnd";
spec[SSL_DOWN_TIKCET_QUERY]="dtkt_query";
}
spec[SSL_NO_CHELLO]="ssl_no_chlo";
spec[SSL_NO_SNI]="ssl_no_sni";
spec[SSL_FAKE_CRT]="ssl_fk_crt";
spec[KEY_KEEPER_ASK]="kyr_ask";
spec[KEY_KEEPER_ISSUE]="kyr_new";
spec[KEY_KEEPER_CACHE_SIZE]="kyr_cache";
spec[SSL_SVC_PINNING]="ssl_pinning";
spec[SSL_SVC_MAUTH]="ssl_mauth";
spec[SSL_SVC_CT_CERT]="ssl_ct_crt";
spec[SSL_SVC_EV_CERT]="ssl_ev_crt";
spec[SSL_SVC_APP_NOT_PINNING]="app_no_pinning";
spec[SSL_SVC_TRUSTED_CERT]="trusted_cert_nums";
for(i=0;i<SSL_STAT_MAX;i++)
{
if(spec[i]!=NULL)
{
mgr->fs_id[i]=FS_register(mgr->fs_handle, FS_STYLE_FIELD, FS_CALC_CURRENT,spec[i]);
}
}
int value=mgr->fs_id[SSL_UP_CACHE_HIT];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
value=mgr->fs_id[SSL_UP_CACHE_QUERY];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(mgr->fs_handle,
mgr->fs_id[SSL_UP_CACHE_HIT],
mgr->fs_id[SSL_UP_CACHE_QUERY],
1,
FS_STYLE_STATUS,
FS_CALC_CURRENT,
"usess_hit");
value=mgr->fs_id[SSL_DOWN_CACHE_HIT];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
value=mgr->fs_id[SSL_DOWN_CACHE_QUERY];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(mgr->fs_handle,
mgr->fs_id[SSL_DOWN_CACHE_HIT],
mgr->fs_id[SSL_DOWN_CACHE_QUERY],
1,
FS_STYLE_STATUS,
FS_CALC_CURRENT,
"dsess_hit");
if(!mgr->no_sessticket)
{
value=mgr->fs_id[SSL_DOWN_TIKCET_QUERY];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
value=mgr->fs_id[SSL_DOWN_TICKET_REUSE];
FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(mgr->fs_handle,
mgr->fs_id[SSL_DOWN_TICKET_REUSE],
mgr->fs_id[SSL_DOWN_TIKCET_QUERY],
1,
FS_STYLE_STATUS,
FS_CALC_CURRENT,
"dtkt_hit");
}
struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here.
mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr);
evtimer_add(mgr->gcev, &gc_delay);
return;
}
static void downstream_ossl_init(struct ssl_stream* s_stream);
static int upstream_ossl_init(struct ssl_stream* s_stream);
static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr);
struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result)
{
struct peek_client_hello_ctx* ctx= (struct peek_client_hello_ctx*) result;
struct ssl_chello * p = ctx->chello;
ctx->chello=NULL;
return p;
}
struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir,
struct ssl_chello * client_hello, struct keyring * kyr, struct ssl_stream * peer, struct tfe_stream* tcp_stream)
{
UNUSED int ret = 0;
struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1);
s_stream->dir = dir;
s_stream->mgr = mgr;
s_stream->_do_not_use.fd = fd;
s_stream->ssl_max_version=mgr->ssl_max_version;
s_stream->ssl_min_version=mgr->ssl_min_version;
s_stream->peer=peer;
s_stream->tcp_stream=tcp_stream;
ret = getpeername(fd, (struct sockaddr *) (&s_stream->peer_addr), &(s_stream->peer_addrlen));
switch (dir)
{
case CONN_DIR_DOWNSTREAM:
assert(peer!=NULL);
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_NEW]));
s_stream->down_parts.keyring = kyr;
s_stream->peer=peer;
break;
case CONN_DIR_UPSTREAM:
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_NEW]));
s_stream->up_parts.verify_param.no_verify_cn=1;
s_stream->up_parts.client_hello = client_hello;
s_stream->ssl_min_version=client_hello->min_version.ossl_format;
s_stream->ssl_max_version=client_hello->max_version.ossl_format;
break;
default: assert(0);
}
return s_stream;
}
static void ssl_stream_free(struct ssl_stream * s_stream)
{
if(s_stream->ssl)
{
SSL_free(s_stream->ssl);
s_stream->ssl = NULL;
}
switch (s_stream->dir)
{
case CONN_DIR_DOWNSTREAM:
if (s_stream->down_parts.keyring != NULL)
{
key_keeper_free_keyring(s_stream->down_parts.keyring);
s_stream->down_parts.keyring = NULL;
}
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED]));
break;
case CONN_DIR_UPSTREAM:
if (s_stream->up_parts.client_hello != NULL)
{
ssl_chello_free(s_stream->up_parts.client_hello);
s_stream->up_parts.client_hello = NULL;
}
ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_CLOSED]));
break;
default: assert(0);
}
if(s_stream->alpn_selected)
{
s_stream->alpn_selected=NULL;
}
s_stream->mgr = NULL;
free(s_stream);
return;
}
static void log_ssl_master_key(struct ssl_stream *s_stream, tfe_conn_dir dir, FILE* fp)
{
char* key_str=NULL;
key_str=ssl_ssl_masterkey_to_str(s_stream->ssl);
char time_str[TFE_SYMBOL_MAX];
time_t now=time(NULL);
tfe_thread_safe_ctime(&now, time_str, sizeof(time_str));
fprintf(fp, "#%s %s %s\n%s\n", time_str, tfe_stream_conn_dir_to_str(dir), s_stream->tcp_stream->str_stream_info ? s_stream->tcp_stream->str_stream_info : NULL, key_str);
free(key_str);
return;
}
void ssl_manager_destroy(struct ssl_mgr * mgr)
{
if (mgr->key_keeper)
{
key_keeper_destroy(mgr->key_keeper);
}
if (mgr->trust_CA_store)
{
ssl_trusted_cert_storage_destroy(mgr->trust_CA_store);
mgr->trust_CA_store = NULL;
}
if(mgr->down_sess_cache)
{
ssl_sess_cache_destroy(mgr->down_sess_cache);
}
if(mgr->up_sess_cache)
{
ssl_sess_cache_destroy(mgr->up_sess_cache);
}
if(mgr->gcev)
{
event_free(mgr->gcev);
}
if(mgr->fp_master_key)
{
fclose(mgr->fp_master_key);
}
free(mgr);
}
struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section,
struct event_base * ev_base_gc, struct key_keeper * key_keeper, void * logger)
{
unsigned int stek_group_num = 0;
unsigned int stek_rotation_time = 0;
struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1);
int ret = 0;
char ja3_table_name[TFE_STRING_MAX] = {0};
char version_str[TFE_SYMBOL_MAX] = {};
mgr->logger = logger;
mgr->ev_base_gc=ev_base_gc;
if (ssl_mid_cert_kafka_logger_create(ini_profile, section))
{
goto error_out;
}
MESA_load_profile_uint_def(ini_profile, section, "ssl_debug", &(ssl_debug), 0);
MESA_load_profile_string_def(ini_profile, section, "ssl_min_version", version_str, sizeof(version_str), "ssl3");
mgr->ssl_min_version = sslver_str2num(version_str);
if (mgr->ssl_min_version < 0)
{
TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str);
goto error_out;
}
MESA_load_profile_string_def(ini_profile, section, "ssl_max_version", version_str, sizeof(version_str), "tls12");
mgr->ssl_max_version = sslver_str2num(version_str);
if (mgr->ssl_max_version < 0)
{
TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str);
goto error_out;
}
ret=ssl_init();
if(ret<0)
{
TFE_LOG_ERROR(logger, "OpenSSL global init failed.");
goto error_out;
}
//tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0.
mgr->sslmethod = TLS_method;
SSL_CTX_EX_DATA_IDX_SSLMGR = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL);
SSL_EX_DATA_IDX_SSLSTREAM = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL);
MESA_load_profile_uint_def(ini_profile, section, "ssl_compression", &(mgr->sslcomp), 1);
MESA_load_profile_uint_def(ini_profile, section, "no_ssl2", &(mgr->no_ssl2), 1);
MESA_load_profile_uint_def(ini_profile, section, "no_ssl3", &(mgr->no_ssl3), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0);
MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers,
sizeof(mgr->default_ciphers), DFLT_CIPHERS);
MESA_load_profile_uint_def(ini_profile, section, "no_session_cache", &(mgr->no_sesscache), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_session_ticket", &(mgr->no_sessticket), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_alpn", &(mgr->no_alpn), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_cert_verify", &(mgr->no_cert_verify), 0);
MESA_load_profile_uint_def(ini_profile, section, "no_mirror_client_cipher_suite",
&(mgr->no_mirror_client_cipher_suite), 0);
MESA_load_profile_uint_def(ini_profile, section, "session_cache_slots",
&(mgr->sess_cache_slots), 4 * 1024 * 1024);
MESA_load_profile_uint_def(ini_profile, section, "session_cache_expire_seconds",
&(mgr->sess_expire_seconds), 30 * 60);
if(!mgr->no_sesscache)
{
mgr->up_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM);
mgr->down_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM);
}
MESA_load_profile_uint_def(ini_profile, section, "stek_group_num", &stek_group_num, 1);
MESA_load_profile_uint_def(ini_profile, section, "stek_rotation_time", &stek_rotation_time, 3600);
if(!mgr->no_sessticket)
{
mgr->down_stek_box = sess_ticket_box_create(ev_base_gc, stek_group_num, stek_rotation_time, logger);
}
MESA_load_profile_uint_def(ini_profile, section, "service_cache_slots",
&(mgr->svc_cache_slots), 4 * 1024 * 1024);
MESA_load_profile_uint_def(ini_profile, section, "service_cache_expire_seconds",
&(mgr->svc_expire_seconds), 5 * 60);
MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_as_pinning_cnt",
&(mgr->svc_fail_as_pinning_cnt), 4);
MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_as_proto_err_cnt",
&(mgr->svc_fail_as_proto_err_cnt), 5);
MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_time_window",
&(mgr->svc_cnt_time_window), 30);
MESA_load_profile_string_def(ini_profile, section, "ssl_ja3_table", ja3_table_name, sizeof(ja3_table_name), "PXY_SSL_FINGERPRINT");
mgr->svc_cache=ssl_service_cache_create(mgr->svc_cache_slots, mgr->svc_expire_seconds,
mgr->svc_fail_as_pinning_cnt,
mgr->svc_fail_as_proto_err_cnt,
mgr->svc_cnt_time_window, ja3_table_name);
if (mgr->svc_cache == NULL)
{
TFE_LOG_ERROR(logger, "Failed to create service cache");
goto error_out;
}
mgr->key_keeper = key_keeper;
MESA_load_profile_uint_def(ini_profile, section, "trusted_cert_load_local",
&(mgr->trusted_cert_load_local), 1);
MESA_load_profile_uint_def(ini_profile, section, "check_cert_crl", &(mgr->cert_verify_param.check_crl), 0);
if(mgr->trusted_cert_load_local)//Other wise, use policy defined trusted CA file.
{
MESA_load_profile_string_def(ini_profile, section, "trusted_cert_file", mgr->trusted_cert_file, sizeof(mgr->trusted_cert_file),
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem");
MESA_load_profile_string_def(ini_profile, section, "trusted_cert_dir", mgr->trusted_cert_dir, sizeof(mgr->trusted_cert_dir),
"./resource/tfe/trusted_storage");
}
mgr->trust_CA_store = ssl_trusted_cert_storage_create(mgr->trusted_cert_file, mgr->trusted_cert_dir, &(mgr->cert_verify_param));
if (mgr->trust_CA_store == NULL)
{
TFE_LOG_ERROR(logger, "Failed at creating X509_STORE");
goto error_out;
}
memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context));
MESA_load_profile_uint_def(ini_profile, section, "log_master_key", &(mgr->log_master_key), 0);
MESA_load_profile_string_def(ini_profile, section, "key_log_file", mgr->master_key_file, sizeof(mgr->master_key_file),
"./sslkeylog.log");
if(mgr->log_master_key)
{
mgr->fp_master_key=fopen(mgr->master_key_file, "a");
if(mgr->fp_master_key==NULL)
{
TFE_LOG_ERROR(logger, "Failed at open master key log file %s", mgr->master_key_file);
mgr->log_master_key=0;
}
}
mgr->fs_handle=tfe_proxy_get_fs_handle();
ssl_stat_init(mgr);
return mgr;
error_out:
ssl_manager_destroy(mgr);
return NULL;
}
void ssl_manager_set_new_upstream_cb(struct ssl_mgr * mgr, ssl_stream_new_hook* new_upstream_cb, void* u_para)
{
mgr->on_new_upstream_cb=new_upstream_cb;
mgr->upstream_cb_param=u_para;
return;
}
void peek_client_hello_ctx_free(struct peek_client_hello_ctx * _ctx)
{
event_free(_ctx->ev);
_ctx->ev = NULL;
if(_ctx->chello!=NULL)
{
ssl_chello_free(_ctx->chello);
_ctx->chello=NULL;
}
free(_ctx);
}
void peek_client_hello_ctx_free_cb(void * p)
{
struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)p;
return peek_client_hello_ctx_free(_ctx);
}
static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg)
{
struct promise * promise = (struct promise *) arg;
//use promise_get_ctx instead of promise_dettach_ctx for try more times.
struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *) promise_get_ctx(promise);
enum chello_parse_result chello_status=CHELLO_PARSE_INVALID_FORMAT;
struct ssl_chello* chello=NULL;
const char * reason = NULL;
unsigned char buf[2048];
ssize_t n = 0;
n = recv(fd, buf, sizeof(buf), MSG_PEEK);
if (n == -1)
{
char* addr_string=tfe_string_addr_create_by_fd(fd, CONN_DIR_DOWNSTREAM);
TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection, addr_string is %s, errno is %d, errmsg is %s\n", addr_string, errno, strerror(errno));
/* after log, reset errno */
errno = 0;
free(addr_string);
goto failed;
}
if (n == 0)
{
goto failed;
}
chello=ssl_chello_parse(buf, n, &chello_status);
switch(chello_status)
{
case CHELLO_PARSE_SUCCESS:
{
if (is_ssl_debug())
{
char *addr = tfe_string_addr_create_by_fd(fd, CONN_DIR_DOWNSTREAM);
struct ssl_ja3 *fingerprint = ssl_ja3_generate_fingerprint(buf, n);
TFE_LOG_INFO(ctx->logger, "ja3:%s, hash:%s addr:%s\n", fingerprint ? fingerprint->ja3 : "null", fingerprint ? fingerprint->ja3_hash : "null", addr);
ssl_ja3_free(fingerprint);
free(addr);
}
promise_dettach_ctx(promise);
ctx->chello=chello;
promise_success(promise, ctx);
peek_client_hello_ctx_free(ctx);
break;
}
case CHELLO_PARSE_NOT_ENOUGH_BUFF:
{
ssl_chello_free(chello);
chello=NULL;
if (ctx->sni_peek_retries++ > MAX_NET_RETRIES)
{
TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n");
reason = "too many peek retries";
goto failed;
}
/* ssl_tls_clienthello_parse indicates that we
* should retry later when we have more data, and we
* haven't reached the maximum retry count yet.
* Reschedule this event as timeout-only event in
* order to prevent busy looping over the read event.
* Because we only peeked at the pending bytes and
* never actually read them, fd is still ready for
* reading now. We use 25 * 0.2 s = 5 s timeout. */
struct timeval retry_delay = {0, 100};
event_del(ctx->ev);
event_free(ctx->ev);
ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise);
assert(ctx->ev != NULL);
event_add(ctx->ev, &retry_delay);
break;
}
case CHELLO_PARSE_INVALID_FORMAT:
{
ssl_chello_free(chello);
chello=NULL;
TFE_LOG_ERROR(ctx->logger,
"Peeking did not yield a (truncated) ClientHello message, aborting connection\n");
reason = "see no client hello";
goto failed;
break;
}
default:
assert(0);
}
return;
failed:
promise_dettach_ctx(promise);
promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason);
peek_client_hello_ctx_free(ctx);
return;
}
static void ssl_async_peek_client_hello(struct future * f, evutil_socket_t fd, struct event_base * evbase,
void * logger)
{
struct promise * p = future_to_promise(f);
struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1);
ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p);
ctx->evbase = evbase;
ctx->logger = logger;
promise_set_ctx(p, (void *) ctx, peek_client_hello_ctx_free_cb);
event_add(ctx->ev, NULL);
return;
}
int ossl_client_cert_cb(SSL *ssl, X509 **x509, EVP_PKEY **pkey)
{
struct ssl_stream* s_upstream=NULL;
s_upstream=(struct ssl_stream*)SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM);
s_upstream->up_parts.svc_status.is_mutual_auth=1;
ssl_service_cache_write(s_upstream->mgr->svc_cache, s_upstream->up_parts.client_hello, s_upstream->tcp_stream, &s_upstream->up_parts.svc_status);
return 0;
}
/*
* Create new SSL context for outgoing connections to the original destination.
* If hostname sni is provided, use it for Server Name Indication.
*/
static int upstream_ossl_init(struct ssl_stream* s_stream)
{
SSL_CTX * sslctx = NULL;
SSL * ssl = NULL;
SSL_SESSION * sess = NULL;
struct ssl_mgr * mgr = s_stream->mgr;
struct ssl_chello * chello=s_stream->up_parts.client_hello;
sslctx = SSL_CTX_new(mgr->sslmethod());
if (sslctx == NULL)
{
TFE_LOG_ERROR(mgr->logger, "ssl ctx new failed.");
return -1;
}
sslctx_set_opts(sslctx, mgr);
int ret=0;
char common_cipher[TFE_STRING_MAX]={0}, tls13_cipher[TFE_STRING_MAX]={0};
if(chello->cipher_suites)
{
ssl_cipher_suites_to_name(chello->cipher_suites, chello->cipher_suites_len,
common_cipher, sizeof(common_cipher), tls13_cipher, sizeof(tls13_cipher));
}
if(strlen(common_cipher)>0)
{
//SSL_CTX_set_cipher_list() and SSL_set_cipher_list() return 1 if any cipher could be selected and 0 on complete failure.
ret=SSL_CTX_set_cipher_list(sslctx, common_cipher);
if(ret==0)
{
TFE_LOG_ERROR(mgr->logger, "ssl ctx set cipher list %s failed.", common_cipher);
SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers);
}
}
else
{
ret=SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers);
}
if(strlen(tls13_cipher)>0 && s_stream->ssl_max_version==TLS1_3_VERSION)
{
//SSL_CTX_set_ciphersuites(sslctx, tls13_cipher);
}
if (SSL_CTX_set_min_proto_version(sslctx, s_stream->ssl_min_version) == 0)
{
SSL_CTX_free(sslctx);
TFE_LOG_ERROR(mgr->logger, "ssl set min proto version failed %d.", s_stream->ssl_min_version);
return -1;
}
if (SSL_CTX_set_max_proto_version(sslctx, s_stream->ssl_max_version) == 0)
{
SSL_CTX_free(sslctx);
TFE_LOG_ERROR(mgr->logger, "ssl set max proto version failed %d.", s_stream->ssl_max_version);
return -1;
}
SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL);
SSL_CTX_set_client_cert_cb(sslctx, ossl_client_cert_cb);
if(mgr->no_sesscache)
{
SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_OFF);
}
else
{
SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_NO_INTERNAL);
/* session resuming based on remote endpoint address and port */
sess = up_session_get(mgr->up_sess_cache,
(struct sockaddr *) &(s_stream->peer_addr), s_stream->peer_addrlen, chello->sni,
s_stream->ssl_min_version, s_stream->ssl_max_version);
if (sess)
{
ret=SSL_CTX_add_session(sslctx, sess); /* increments sess refcount */
assert(ret==1);
SSL_SESSION_free(sess);
}
}
ssl = SSL_new(sslctx);
SSL_CTX_free(sslctx); /* SSL_new() increments refcount */
if (!ssl)
{
TFE_LOG_ERROR(mgr->logger, "ssl new failed.");
return -1;
}
SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream);
if (chello->sni)
{
SSL_set_tlsext_host_name(ssl, chello->sni);
}
if (chello->alpn && s_stream->up_parts.apln_enabled)
{
ret=SSL_set_alpn_protos(ssl, (unsigned char*)chello->alpn, strlen(chello->alpn));
assert(ret==0);
}
/* lower memory footprint for idle connections */
SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS);
s_stream->ssl=ssl;
return 0;
}
void ssl_connect_server_ctx_free(struct ssl_connect_server_ctx * ctx)
{
if (ctx->s_stream != NULL)
{
ssl_stream_free(ctx->s_stream);
}
if (ctx->bev != NULL)
{
bufferevent_free(ctx->bev);
ctx->bev = NULL;
}
if (ctx->f_peek_chello != NULL)
{
future_destroy(ctx->f_peek_chello);
ctx->f_peek_chello = NULL;
}
free(ctx);
return;
}
void wrap_ssl_connect_server_ctx_free(void *p)
{
struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *)p;
ssl_connect_server_ctx_free(ctx);
}
enum ssl_stream_action ssl_upstream_create_result_release_action(future_result_t * result)
{
struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
return ctx->s_stream->up_parts.action;
}
evutil_socket_t ssl_upstream_create_result_release_fd(future_result_t * result)
{
struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
return ctx->fd_upstream;
}
struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result)
{
struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
struct ssl_stream * ret = ctx->s_stream;
ctx->s_stream = NULL; //giveup ownership
return ret;
}
struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result)
{
struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
struct bufferevent * ret = ctx->bev;
ctx->bev = NULL; //giveup ownership
return ret;
}
const char* ssl_stream_dump_info(struct ssl_stream *stream, char* buffer, size_t sz)
{
snprintf(buffer, sz, "%s:%s", SSL_get_version(stream->ssl),
stream->dir==CONN_DIR_UPSTREAM ? stream->up_parts.client_hello->sni:NULL);
return buffer;
}
void ssl_stream_set_cmsg_string(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, const char* value_str)
{
struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream);
UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)value_str, (uint16_t)strlen(value_str));
assert(ret==0);
return;
}
static void ssl_stream_set_cmsg_uint8(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, uint8_t value_int)
{
struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream);
UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)&value_int, (uint16_t)sizeof(value_int));
assert(ret==0);
return;
}
static void ssl_stream_set_cmsg_integer(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, uint64_t value_int)
{
struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream);
UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)&value_int, (uint16_t)sizeof(value_int));
assert(ret==0);
return;
}
unsigned long ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struct ssl_mgr* mgr)
{
unsigned long sslerr=0, ret_sslerr=0;
int fd=bufferevent_getfd(bev);
char* addr_string=tfe_string_addr_create_by_fd(fd, dir);
void* logger=mgr->logger;
int fs_id=-1;
/* Can happen for socket errs, ssl errs;
* may happen for unclean ssl socket shutdowns. */
sslerr = bufferevent_get_openssl_error(bev);
ret_sslerr=ERR_GET_REASON(sslerr);
switch(ret_sslerr)
{
case SSL_R_INAPPROPRIATE_FALLBACK:
if(dir==CONN_DIR_DOWNSTREAM) fs_id=SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK;
break;
case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN:
if(dir==CONN_DIR_DOWNSTREAM) fs_id=SSL_DOWN_ERR_NO_CERT;
break;
case SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM:
case SSL_R_UNSUPPORTED_PROTOCOL:
case SSL_R_UNSUPPORTED_SSL_VERSION:
case SSL_R_UNSUPPORTED_STATUS_TYPE:
case SSL_R_NO_PROTOCOLS_AVAILABLE:
if(dir==CONN_DIR_UPSTREAM) fs_id=SSL_UP_ERR_UNSUPPORT_PROTO;
case SSL_R_NO_CIPHERS_AVAILABLE:
if(dir==CONN_DIR_UPSTREAM) fs_id=SSL_UP_ERR_NO_CIPHER;
break;
default:
fs_id=-1;
}
if(fs_id>=0)
{
mgr->stat_val[fs_id]++;
}
if (!errno && !sslerr)
{
/* We have disabled notification for unclean shutdowns
* so this should not happen; log a warning. */
TFE_LOG_ERROR(logger,"Warning: Spurious error from "
"bufferevent (errno=0, sslerr=0)\n");
}
else if (ERR_GET_REASON(sslerr) ==
SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE)
{
/* these can happen due to client cert auth,
* only log error if debugging is activated */
TFE_LOG_ERROR(logger,"Handshake Error from bufferevent of ssl %s %s: "
"%i:%s %lu:%i:%s:%i:%s:%i:%s",
tfe_stream_conn_dir_to_str(dir),
addr_string,
errno,
errno ? strerror(errno) : "-",
sslerr,
ERR_GET_REASON(sslerr),
sslerr ?
ERR_reason_error_string(sslerr) : "-",
ERR_GET_LIB(sslerr),
sslerr ?
ERR_lib_error_string(sslerr) : "-",
ERR_GET_FUNC(sslerr),
sslerr ?
ERR_func_error_string(sslerr) : "-");
/* after log, reset errno */
errno = 0;
while ((sslerr = bufferevent_get_openssl_error(bev)))
{
TFE_LOG_ERROR(logger,"Additional SSL error: "
"%lu:%i:%s:%i:%s:%i:%s",
sslerr,
ERR_GET_REASON(sslerr),
ERR_reason_error_string(sslerr),
ERR_GET_LIB(sslerr),
ERR_lib_error_string(sslerr),
ERR_GET_FUNC(sslerr),
ERR_func_error_string(sslerr));
}
}
else
{
/* real errors */
TFE_LOG_ERROR(logger,"Error from bufferevent of ssl %s %s: "
"%i:%s %lu:%i:%s:%i:%s:%i:%s",
tfe_stream_conn_dir_to_str(dir),
addr_string,
errno,
errno ? strerror(errno) : "-",
sslerr,
ERR_GET_REASON(sslerr),
sslerr ?
ERR_reason_error_string(sslerr) : "-",
ERR_GET_LIB(sslerr),
sslerr ?
ERR_lib_error_string(sslerr) : "-",
ERR_GET_FUNC(sslerr),
sslerr ?
ERR_func_error_string(sslerr) : "-");
/* after log, reset errno */
errno = 0;
while ((sslerr = bufferevent_get_openssl_error(bev)))
{
TFE_LOG_ERROR(logger,"Additional SSL error: "
"%lu:%i:%s:%i:%s:%i:%s\n",
sslerr,
ERR_GET_REASON(sslerr),
ERR_reason_error_string(sslerr),
ERR_GET_LIB(sslerr),
ERR_lib_error_string(sslerr),
ERR_GET_FUNC(sslerr),
ERR_func_error_string(sslerr));
}
}
free(addr_string);
return ret_sslerr;
}
void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr, struct ssl_mgr* mgr)
{
struct ssl_upstream_parts* s_upstream=NULL;
assert(sslerr);
switch(s_stream->dir)
{
case CONN_DIR_DOWNSTREAM:
s_upstream= &(s_stream->peer->up_parts);
if( (sslerr==SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN ||
sslerr==SSL_R_SSLV3_ALERT_BAD_CERTIFICATE ||
sslerr==SSL_R_TLSV1_ALERT_UNKNOWN_CA ||
sslerr==SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE ||
sslerr==SSL_R_UNKNOWN_CERTIFICATE_TYPE)
&& s_upstream->is_server_cert_verify_passed
&& s_upstream->verify_result.is_hostmatched)
{
s_upstream->svc_status.cli_pinning_status=PINNING_ST_PINNING;
// This feature is also displayed when the root certificate is not installed.
// Here no longer set the pinning state to cmsg, but use the pinng state
// corrected by app_is_not_pinnig in the peek_chello_on_succ function
// ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING);
ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status);
}
else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN)
{
s_upstream->svc_status.has_protocol_errors=1;
ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status);
}
break;
case CONN_DIR_UPSTREAM:
s_upstream=&(s_stream->up_parts);
s_upstream->svc_status.has_protocol_errors=1;
ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->up_parts.svc_status));
break;
default:
assert(0);
}
return;
}
void ssl_stream_process_zero_eof(struct ssl_stream * s_stream, struct ssl_mgr* mgr)
{
struct ssl_upstream_parts* s_upstream=NULL;
if(s_stream->dir==CONN_DIR_UPSTREAM)
{
return;
}
assert(mgr==s_stream->mgr);
s_upstream=&s_stream->peer->up_parts;
if(s_upstream->verify_result.is_hostmatched && s_upstream->is_server_cert_verify_passed )
{
const char *sni = s_upstream->client_hello ? (s_upstream->client_hello->sni ? s_upstream->client_hello->sni : "null"): "null";
TFE_LOG_DEBUG(mgr->logger, "sni:%s cert verify passed and hit zero eof, set protocol errors", sni);
s_upstream->svc_status.has_protocol_errors=1;
ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->peer->up_parts.svc_status));
}
s_stream->error=SSL_STREAM_R_CLIENT_CLOSED;
return;
}
/*
* Callback for meta events on the up- and downstream connection bufferevents.
* Called when EOF has been reached, a connection has been made, and on errors.
*/
static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, void * arg)
{
struct promise * p = (struct promise *) arg;
struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p);
struct ssl_stream * s_stream = ctx->s_stream;
assert(s_stream->dir==CONN_DIR_UPSTREAM);
struct ssl_upstream_parts* s_upstream=&(s_stream->up_parts);
struct ssl_mgr* mgr=s_stream->mgr;
SSL_SESSION * ssl_sess = NULL;
char error_str[TFE_STRING_MAX];
uint64_t jiffies_ms;
unsigned long sslerr=0;
if (events & BEV_EVENT_ERROR)
{
ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
sslerr=ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, ctx->mgr);
if(sslerr)
{
ssl_stream_process_error(s_stream, sslerr, mgr);
}
s_stream->error=SSL_STREAM_R_SERVER_PROTOCOL_ERROR;
}
else if(events & BEV_EVENT_EOF)
{
ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
s_stream->error=SSL_STREAM_R_SERVER_CLOSED;
}
else if(events & BEV_EVENT_TIMEOUT)
{
ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
s_stream->error=SSL_STREAM_R_CONNECT_SERVER_TIMEOUT;
}
else if(events & BEV_EVENT_CONNECTED)
{
bufferevent_disable(ctx->bev, EV_READ | EV_WRITE);
bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success
clock_gettime(CLOCK_MONOTONIC, &(ctx->end));
jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000;
if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS)
{
TFE_LOG_INFO(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s",
jiffies_ms,
s_stream->tcp_stream->str_stream_info,
s_upstream->client_hello->sni);
}
s_stream->connect_latency_ms=jiffies_ms;
ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_SERVER_SIDE_LATENCY, jiffies_ms);
if(!SSL_session_reused(s_stream->ssl))
{
if(mgr->no_cert_verify)
{
s_upstream->is_server_cert_verify_passed=1;
}
else
{
s_upstream->is_server_cert_verify_passed = !!ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store,
s_stream->ssl, s_stream->up_parts.client_hello->sni, &(s_stream->up_parts.verify_param),
error_str, sizeof(error_str), &(s_stream->up_parts.verify_result));
TFE_LOG_DEBUG(g_default_logger,
"addr:%s, sni:%s, is_cert_verify_passed:%d, cet_real_untrust:%d, verify_host_fail:%d, verify_issure_fail:%d, verify_self_signed_fail:%d, verify_expiry_date_fail:%d, verify_other_fail:%d, %s",
s_stream->tcp_stream->str_stream_info,
s_upstream->client_hello->sni,
s_upstream->is_server_cert_verify_passed,
((s_upstream->verify_param.real_untrust & 0xff) ? 1 : 0),
((s_upstream->verify_param.real_untrust & 0x01) ? 1 : 0),
((s_upstream->verify_param.real_untrust & 0x02) ? 1 : 0),
((s_upstream->verify_param.real_untrust & 0x04) ? 1 : 0),
((s_upstream->verify_param.real_untrust & 0x08) ? 1 : 0),
((s_upstream->verify_param.real_untrust & 0x10) ? 1 : 0),
(s_upstream->is_server_cert_verify_passed == 0 ? error_str : ""));
s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct;
s_upstream->svc_status.is_ev=s_upstream->verify_result.is_ev;
ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &(s_upstream->svc_status));
TFE_LOG_DEBUG(mgr->logger, "stream:%s sni:%s hostmatch:%d, ct:%d, ev:%d",
s_stream->tcp_stream->str_stream_info,
s_upstream->client_hello->sni,
s_upstream->verify_result.is_hostmatched,
s_upstream->verify_result.is_ct,
s_upstream->verify_result.is_ev);
if((!s_upstream->is_server_cert_verify_passed || !s_upstream->verify_result.is_hostmatched) && s_upstream->block_fake_cert)
{
s_stream->up_parts.action=SSL_ACTION_SHUTDOWN;
}
}
if(s_upstream->is_server_cert_verify_passed)
{
if(!mgr->no_sesscache && s_stream->up_parts.action==SSL_ACTION_INTERCEPT)
{
//ONLY verified session is cacheable.
//The reference count of the SSL_SESSION is not incremented, so no need to free.
ssl_sess = SSL_get0_session(s_stream->ssl);
up_session_set(mgr->up_sess_cache, (struct sockaddr *)&(ctx->addr),
ctx->addrlen, s_upstream->client_hello->sni, SSL_version(s_stream->ssl), ssl_sess);
}
}
else
{
ATOMIC_INC(&(mgr->stat_val[SSL_FAKE_CRT]));
char* addr_str=tfe_string_addr_create_by_fd(ctx->fd_upstream, CONN_DIR_UPSTREAM);
TFE_LOG_INFO(mgr->logger, "Fake Cert %s %s : %s", addr_str, s_upstream->client_hello->sni, error_str);
free(addr_str);
}
}
else
{
//Do not perform cert verification on reused session.
s_upstream->is_server_cert_verify_passed=1;
}
if(mgr->log_master_key)
{
log_ssl_master_key(s_stream, CONN_DIR_UPSTREAM, mgr->fp_master_key);
}
const unsigned char *alpn_data=NULL;
unsigned int alpn_len=0;
SSL_get0_alpn_selected(s_stream->ssl, &alpn_data, &alpn_len);
if(alpn_len>0&&alpn_data!=NULL)
{
if(0==memcmp(alpn_data, SSL_ALPN_HTTP_1_1+1, alpn_len))
{
s_stream->alpn_selected=SSL_ALPN_HTTP_1_1;
}
else if(0==memcmp(alpn_data, SSL_ALPN_HTTP_2+1, alpn_len))
{
s_stream->alpn_selected=SSL_ALPN_HTTP_2;
}
else
{
s_stream->alpn_selected=NULL;
}
}
s_stream->negotiated_version=SSL_version(s_stream->ssl);
ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_SERVER_SIDE_VERSION, SSL_get_version(s_stream->ssl));
ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_CERT_VERIFY, s_upstream->is_server_cert_verify_passed);
promise_success(p, ctx);
}
if(s_stream->error)
{
ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error));
snprintf(error_str, sizeof(error_str), "%s, sni=%s", ssl_stream_get_error_string(s_stream->error), s_upstream->client_hello->sni);
promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str);
}
wrap_ssl_connect_server_ctx_free(ctx);
return;
}
static void peek_chello_on_succ(future_result_t * result, void * user)
{
struct promise * p = (struct promise *) user;
struct ssl_connect_server_ctx* ctx = (struct ssl_connect_server_ctx *) promise_get_ctx(p);
struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id);
struct ssl_stream* s_stream=NULL;
struct ssl_chello* chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream.
if(chello->sni==NULL)
{
ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_SNI]));
}
int ret=0;
struct ssl_service_status* svc_status=NULL;
clock_gettime(CLOCK_MONOTONIC, &(ctx->start));
s_stream= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL, NULL, ctx->tcp_stream);
svc_status=&s_stream->up_parts.svc_status;
ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, s_stream->tcp_stream, svc_status);
if(ret==1)
{
TFE_LOG_DEBUG(ctx->mgr->logger, "%s %s service status pinning:%d, mauth:%d, err:%d, ct:%d, ev:%d, ja3_pinning_status:%d",
s_stream->tcp_stream->str_stream_info,
chello->sni,
svc_status->cli_pinning_status,
svc_status->is_mutual_auth,
svc_status->has_protocol_errors,
svc_status->is_ct,
svc_status->is_ev,
svc_status->ja3_pinning_status);
}
switch (svc_status->ja3_pinning_status)
{
case JA3_PINNING_STATUS_NOT_PINNING:
ctx->mgr->svc_cache->stat.app_not_pinning_cnt++;
ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_NOT_PINNING);
break;
case JA3_PINNING_STATUS_IS_PINNING:
ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING);
break;
case JA3_PINNING_STATUS_UNKNOWN:
/* fall through */
default:
ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, svc_status->cli_pinning_status);
break;
}
if(ctx->mgr->on_new_upstream_cb)
{
s_stream->up_parts.action=ctx->mgr->on_new_upstream_cb(s_stream, ctx->mgr->upstream_cb_param);
}
else
{
s_stream->up_parts.action=SSL_ACTION_INTERCEPT;
}
if (ATOMIC_READ(&certstore_is_unavailable) > 3)
{
s_stream->up_parts.action=SSL_ACTION_PASSTHROUGH;
ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_PASSTHROUGH_REASON, "Certstore Unavailable");
TFE_LOG_ERROR(ctx->mgr->logger, "CertStore is unavailable, PASSTHROUGH");
}
ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_INTERCEPT_STATE, s_stream->up_parts.action);
ctx->s_stream = s_stream;
if(s_stream->up_parts.action==SSL_ACTION_PASSTHROUGH)
{
promise_dettach_ctx(p);
promise_success(p, ctx);
wrap_ssl_connect_server_ctx_free(ctx);
}
else
{
if (upstream_ossl_init(s_stream))
{
promise_dettach_ctx(p);
promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream ossl init failed");
wrap_ssl_connect_server_ctx_free(ctx);
return;
}
ctx->bev = bufferevent_openssl_socket_new(evbase, ctx->fd_upstream,
ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE );
bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1);
bufferevent_setcb(ctx->bev, NULL, NULL, ssl_server_connected_eventcb, p);
bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only
future_destroy(ctx->f_peek_chello);
ctx->f_peek_chello = NULL;
}
return;
}
static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user)
{
struct promise * p = (struct promise *) user;
struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p);
ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO]));
promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream.");
wrap_ssl_connect_server_ctx_free(ctx);
return;
}
void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream,
evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream)
{
struct promise * p = future_to_promise(f);
struct ssl_connect_server_ctx * ctx = ALLOC(struct ssl_connect_server_ctx, 1);
int ret = 0;
ctx->addrlen = sizeof(ctx->addr);
ret = getpeername(fd_upstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen));
if(ret!=0)
{
ssl_connect_server_ctx_free(ctx);
promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream fd closed");
return;
}
struct event_base* evbase=tfe_proxy_get_work_thread_evbase(tcp_stream->thread_id);
ctx->fd_downstream = fd_downstream;
ctx->fd_upstream = fd_upstream;
ctx->tcp_stream = tcp_stream;
ctx->mgr = mgr;
promise_set_ctx(p, ctx, wrap_ssl_connect_server_ctx_free);
ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p);
ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger);
}
static int ossl_session_ticket_key_callback(SSL *ssl_conn,
unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx,
HMAC_CTX *hctx, int enc)
{
enum STEK_GET_RET ret=STEK_NOT_FOUND;
const EVP_MD *digest=EVP_sha256();
const EVP_CIPHER *cipher=EVP_aes_256_cbc();
size_t size=32;
struct ssl_stream* s_stream = (struct ssl_stream*) SSL_get_ex_data(ssl_conn, SSL_EX_DATA_IDX_SSLSTREAM);
struct ssl_mgr* mgr= s_stream->mgr;
void* logger = s_stream->mgr->logger;
const char * sni = s_stream->peer->up_parts.client_hello->sni;
struct sess_ticket_box * stek_box = s_stream->mgr->down_stek_box;
struct sess_ticket_key cur_stek;
unsigned char buf[33]={0};
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY]));
if (enc == 1)
{
sess_ticket_box_get_key_for_enc(stek_box, sni, &cur_stek);
/* encrypt session stek */
if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1)
{
TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed");
ret=STEK_ERROR;
goto leave;
}
if (EVP_EncryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1)
{
TFE_LOG_ERROR(mgr->logger, "EVP_EncryptInit_ex() failed");
ret=STEK_ERROR;
goto leave;
}
if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1)
{
TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed");
ret=STEK_ERROR;
goto leave;
}
memcpy(name, cur_stek.name, sizeof(cur_stek.name));
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW]));
ret=STEK_FOUND_FRESH;
}
else
{
/* decrypt session stek */
ret=sess_ticket_box_get_key_for_dec(stek_box, sni, name, &cur_stek);
if(ret==STEK_FOUND_FRESH||ret==STEK_FOUND_STALED)
{
if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1)
{
TFE_LOG_ERROR(logger, "HMAC_Init_ex() failed");
ret= STEK_ERROR;
goto leave;
}
if (EVP_DecryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1)
{
TFE_LOG_ERROR(logger, "EVP_DecryptInit_ex() failed");
ret= STEK_ERROR;
goto leave;
}
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_REUSE]));
}
else
{
// full handshake and update the session stek_box
TFE_LOG_INFO(logger, "ssl session stek_box decrypt, key: \"%*s\" not found",(int)(tfe_hexdump(buf, name ,16)-buf), buf);
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND]));
}
}
leave:
return (int)ret;
}
/*
* Called by OpenSSL when a new src SSL session is created.
* OpenSSL increments the refcount before calling the callback and will
* decrement it again if we return 0. Returning 1 will make OpenSSL skip
* the refcount decrementing. In other words, return 0 if we did not
* keep a pointer to the object (which we never do here).
*/
static int ossl_downsess_new_cb(SSL * ssl, SSL_SESSION * sess)
{
struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_CTX_EX_DATA_IDX_SSLMGR);
#ifdef HAVE_SSLV2
/* Session resumption seems to fail for SSLv2 with protocol
* parsing errors, so we disable caching for SSLv2. */
if (SSL_version(ssl) == SSL2_VERSION) {
return 0;
}
#endif /* HAVE_SSLV2 */
if (sess && !mgr->no_sesscache)
{
down_session_set(mgr->down_sess_cache, sess);
}
return 0;
}
/*
* Called by OpenSSL when a src SSL session should be removed.
* OpenSSL calls SSL_SESSION_free() after calling the callback;
* we do not need to free the reference here.
*/
static void ossl_downsess_remove_cb(SSL_CTX * sslctx, SSL_SESSION * sess)
{
struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_CTX_get_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR);
assert(mgr != NULL);
if (sess && !mgr->no_sesscache)
{
down_session_del(mgr->down_sess_cache, sess);
}
return;
}
/*
* Called by OpenSSL when a src SSL session is requested by the client.
OPENSSL_VERSION_NUMBER >= 0x10100000L required.
*/
static SSL_SESSION * ossl_downsess_get_cb(SSL * ssl, const unsigned char * id, int idlen, int * copy)
{
struct ssl_stream* s_stream = (struct ssl_stream*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM);
struct ssl_mgr * mgr = s_stream->mgr;
SSL_SESSION * sess=NULL;
if(!mgr->no_sesscache)
{
*copy = 0; /* SSL should not increment reference count of session */
sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache, id, idlen);
}
return sess;
}
/*
* Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX.
*/
static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr)
{
SSL_CTX_set_options(sslctx, SSL_OP_ALL);
SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG);
SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION);
SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS);
if (mgr->no_ssl2)
{
SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2);
}
if (mgr->no_ssl3)
{
SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3);
}
if (mgr->no_tls10)
{
SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1);
}
if (mgr->no_tls11)
{
SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1);
}
if (mgr->no_tls12)
{
SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2);
}
if(mgr->no_sessticket)
{
SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET);
}
if (!mgr->sslcomp)
{
SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION);
}
}
static int alpn_select_proto_cb(SSL *ssl, const unsigned char **out,
unsigned char *outlen, const unsigned char *in,
unsigned int inlen, void *arg) {
int rv;
const unsigned char* selected_alpn=(const unsigned char*)arg;
(void)ssl;
unsigned char* _out=NULL;
unsigned char _outlen=0;
rv=SSL_select_next_proto(&_out, &_outlen, in, inlen, selected_alpn, strlen((const char*)selected_alpn));
if(rv==OPENSSL_NPN_NEGOTIATED)
{
*out=_out;
*outlen=_outlen;
return SSL_TLSEXT_ERR_OK;
}
else //rv==OPENSSL_NPN_NO_OVERLAP
{
return SSL_TLSEXT_ERR_NOACK;
}
}
/*
* Create and set up a new SSL_CTX instance for terminating SSL.
* Set up all the necessary callbacks, the keyring, the keyring chain and key.
*/
void downstream_ossl_init(struct ssl_stream *s_stream)
{
struct ssl_mgr * mgr=s_stream->mgr;
SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod());
if (!sslctx) return;
struct keyring * crt=s_stream->down_parts.keyring;
int negotiated_version=s_stream->peer->negotiated_version;
const unsigned char* selected_alpn=s_stream->peer->alpn_selected;
SSL * ssl = NULL;
UNUSED int ret = 0;
sslctx_set_opts(sslctx, mgr);
SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers);
//TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L
if(negotiated_version)
{
if (SSL_CTX_set_min_proto_version(sslctx, negotiated_version) == 0 ||
SSL_CTX_set_max_proto_version(sslctx, negotiated_version) == 0)
{
SSL_CTX_free(sslctx);
return;
}
}
else if(mgr->ssl_min_version)
{
if (SSL_CTX_set_min_proto_version(sslctx, mgr->ssl_min_version) == 0 ||
SSL_CTX_set_max_proto_version(sslctx, mgr->ssl_max_version) == 0)
{
SSL_CTX_free(sslctx);
return;
}
}
SSL_CTX_sess_set_new_cb(sslctx, ossl_downsess_new_cb);
SSL_CTX_sess_set_remove_cb(sslctx, ossl_downsess_remove_cb);
SSL_CTX_sess_set_get_cb(sslctx, ossl_downsess_get_cb);
if(!mgr->no_sessticket&&s_stream->peer->up_parts.client_hello->sni)
{
SSL_CTX_set_tlsext_ticket_key_cb(sslctx, ossl_session_ticket_key_callback);
}
SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL);
SSL_CTX_set_session_id_context(sslctx, (const unsigned char *) mgr->ssl_session_context,
sizeof(mgr->ssl_session_context));
ret = SSL_CTX_set_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR, mgr);//used by down session cache get
assert(ret == 1);
if (mgr->dh)
{
SSL_CTX_set_tmp_dh(sslctx, mgr->dh);
}
else
{
SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback);
}
if (mgr->ecdhcurve)
{
EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve);
SSL_CTX_set_tmp_ecdh(sslctx, ecdh);
EC_KEY_free(ecdh);
}
else
{
EC_KEY * ecdh = ssl_ec_by_name(NULL);
SSL_CTX_set_tmp_ecdh(sslctx, ecdh);
EC_KEY_free(ecdh);
}
if(s_stream->peer->up_parts.apln_enabled && selected_alpn)
{
SSL_CTX_set_alpn_select_cb(sslctx, alpn_select_proto_cb, (void*)selected_alpn);
}
SSL_CTX_use_certificate(sslctx, crt->cert);
SSL_CTX_use_PrivateKey(sslctx, crt->key);
for (int i = 0; i < sk_X509_num(crt->chain); i++)
{
X509 * c = sk_X509_value(crt->chain, i);
ssl_x509_refcount_inc(c); /* next call consumes a reference */
SSL_CTX_add_extra_chain_cert(sslctx, c);
}
ssl = SSL_new(sslctx);
SSL_CTX_free(sslctx); // SSL_new() increments refcount
sslctx = NULL;
ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream);
assert(ret == 1);
if (mgr->ssl_mode_release_buffers == 1)
{
/* lower memory footprint for idle connections */
SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS);
}
s_stream->ssl=ssl;
return;
}
void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx)
{
X509_free(ctx->origin_crt);
ctx->origin_crt=NULL;
if (ctx->f_ask_keyring != NULL)
{
future_destroy(ctx->f_ask_keyring);
ctx->f_ask_keyring = NULL;
}
//on success, bev_down and downstream has been transfered to caller by release**
if (ctx->bev_down != NULL)
{
bufferevent_free(ctx->bev_down);
}
if (ctx->downstream != NULL)
{
ssl_stream_free(ctx->downstream);
}
free(ctx);
return;
}
void wrap_ssl_connect_client_ctx_free(void * p)
{
struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *)p;
ssl_connect_client_ctx_free(ctx);
}
struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result)
{
struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result;
struct ssl_stream * ret = ctx->downstream;
ctx->downstream = NULL;
return ret;
}
struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result)
{
struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result;
struct bufferevent * ret = ctx->bev_down;
ctx->bev_down = NULL;
return ret;
}
static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, void * arg)
{
struct promise * p = (struct promise *) arg;
struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p);
struct ssl_stream * s_stream = ctx->downstream;
struct ssl_upstream_parts* s_upstream= &(ctx->peer->up_parts);
struct ssl_mgr* mgr=s_stream->mgr;
char error_str[TFE_STRING_MAX]={0};
uint64_t jiffies_ms=0;
unsigned long sslerr=0;
if (events & BEV_EVENT_ERROR)
{
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
sslerr=ssl_stream_log_error(bev, CONN_DIR_DOWNSTREAM, mgr);
if(sslerr)
{
ssl_stream_process_error(s_stream, sslerr, mgr);
}
s_stream->error=SSL_STREAM_R_CLIENT_PROTOCOL_ERROR;
}
else if(events & BEV_EVENT_EOF)
{
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
ssl_stream_process_zero_eof(s_stream, mgr);
}
else if(events & BEV_EVENT_TIMEOUT)
{
ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
s_stream->error=SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT;
}
else if(events & BEV_EVENT_CONNECTED)
{
clock_gettime(CLOCK_MONOTONIC, &(ctx->end));
jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000;
if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS)
{
TFE_LOG_INFO(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s",
jiffies_ms,
s_stream->tcp_stream->str_stream_info,
s_upstream->client_hello->sni);
}
s_stream->connect_latency_ms=jiffies_ms;
ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_CLIENT_SIDE_LATENCY, jiffies_ms);
bufferevent_disable(ctx->bev_down, EV_READ | EV_WRITE);
bufferevent_setcb(ctx->bev_down, NULL, NULL, NULL, NULL); //leave a clean bev for on_success
if(mgr->log_master_key)
{
log_ssl_master_key(s_stream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key);
}
s_stream->negotiated_version=SSL_version(s_stream->ssl);
ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_CLIENT_SIDE_VERSION, SSL_get_version(s_stream->ssl));
promise_success(p, ctx);
}
if(s_stream->error)
{
ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error));
snprintf(error_str, sizeof(error_str), "%s : sni=%s", ssl_stream_get_error_string(s_stream->error), s_upstream->client_hello->sni);
promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str);
}
ssl_connect_client_ctx_free(ctx);
return;
}
void ask_keyring_on_succ(void * result, void * user)
{
struct promise * p = (struct promise *) user;
struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_get_ctx(p);
struct keyring * kyr = NULL;
struct ssl_mgr * mgr = ctx->ssl_mgr;
struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id);
kyr = key_keeper_release_keyring(result); //kyr will be freed at ssl downstream closing.
clock_gettime(CLOCK_MONOTONIC, &(ctx->start));
ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL,
kyr, ctx->peer, ctx->tcp_stream);
downstream_ossl_init(ctx->downstream);
ctx->bev_down = bufferevent_openssl_socket_new(evbase, ctx->fd_downstream, ctx->downstream->ssl,
BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE);
bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1);
bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p);
bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only
future_destroy(ctx->f_ask_keyring);
ctx->f_ask_keyring = NULL;
}
void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user)
{
struct promise * p = (struct promise *) user;
struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p);
promise_failed(p, error, what);
ssl_connect_client_ctx_free(ctx);
return;
}
/*
* Create a SSL stream for the incoming connection, based on the upstream.
*/
void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream,
evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream)
{
assert(upstream->dir == CONN_DIR_UPSTREAM);
const char* sni=NULL;
struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1);
ctx->ssl_mgr = mgr;
ctx->fd_downstream = fd_downstream;
ctx->tcp_stream = tcp_stream;
struct event_base * evbase=tfe_proxy_get_work_thread_evbase(tcp_stream->thread_id);
struct evdns_base* dnsbase=tfe_proxy_get_work_thread_dnsbase(tcp_stream->thread_id);
struct evhttp_connection *evhttp=tfe_proxy_get_work_thread_evhttp(tcp_stream->thread_id);
if (upstream != NULL)
{
ctx->peer = upstream;
ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl);
sni=upstream->up_parts.client_hello->sni;
}
struct promise * p = future_to_promise(f);
promise_set_ctx(p, ctx, wrap_ssl_connect_client_ctx_free);
ctx->f_ask_keyring = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p);
ctx->is_origin_crt_verify_passed = upstream->up_parts.is_server_cert_verify_passed;
int keyring_id = 0;
if (ctx->is_origin_crt_verify_passed)
{
keyring_id = upstream->up_parts.keyring_for_trusted;
}
else
{
keyring_id = upstream->up_parts.keyring_for_untrusted;
}
key_keeper_async_ask(ctx->f_ask_keyring, mgr->key_keeper, sni, keyring_id, ctx->origin_crt, ctx->is_origin_crt_verify_passed,
evbase, dnsbase, evhttp);
return;
}
/*
* Cleanly shutdown an SSL session on file descriptor fd using low-level
* file descriptor readiness events on event base evbase.
* Guarantees that SSL and the corresponding SSL_CTX are freed and the
* socket is closed, eventually, or in the case of fatal errors, immediately.
*/
void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev)
{
UNUSED struct ssl_shutdown_ctx * sslshutctx = NULL;
evutil_socket_t fd=-1;
fd=bufferevent_getfd(bev);
assert(fd==s_stream->_do_not_use.fd);
unsigned long sslerr=0;
if (s_stream->dir == CONN_DIR_UPSTREAM)
{
size_t rx_offset_this_time = 0;
tfe_stream_info_get(s_stream->tcp_stream, INFO_FROM_UPSTREAM_RX_OFFSET, &rx_offset_this_time, sizeof(rx_offset_this_time));
const char * sni = (s_stream->up_parts.client_hello && s_stream->up_parts.client_hello->sni) ? s_stream->up_parts.client_hello->sni : "null";
TFE_LOG_DEBUG(g_default_logger, "ssl up stream close, rx_offset:%zu, sni:%s", rx_offset_this_time, sni);
}
if(errno)
{
sslerr=ssl_stream_log_error(bev, s_stream->dir, s_stream->mgr);
if(sslerr)
{
ssl_stream_process_error(s_stream, sslerr, s_stream->mgr);
}
}
//sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase);
//pxy_ssl_shutdown_cb(fd, 0, sslshutctx);
bufferevent_setcb(bev, NULL, NULL, NULL, NULL);
SSL_set_shutdown(s_stream->ssl, SSL_RECEIVED_SHUTDOWN);
SSL_shutdown(s_stream->ssl);
bufferevent_disable(bev, EV_READ|EV_WRITE);
struct bufferevent *ubev = bufferevent_get_underlying(bev);
if (ubev) {
bufferevent_disable(ubev, EV_READ|EV_WRITE);
bufferevent_setfd(ubev, -1);
bufferevent_setcb(ubev, NULL, NULL, NULL, NULL);
bufferevent_free(ubev);
}
ssl_stream_free(s_stream);
}
int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file)
{
int ret = ssl_trusted_cert_storage_add(mgr->trust_CA_store, SSL_X509_OBJ_CERT, pem_file);
if (ret == 1)
{
ATOMIC_INC(&((*(mgr->svc_cache)).stat.trusted_cert_cnt));
}
return ret;
}
int ssl_manager_del_trust_ca(struct ssl_mgr* mgr, const char* pem_file)
{
int ret = ssl_trusted_cert_storage_del(mgr->trust_CA_store, SSL_X509_OBJ_CERT, pem_file);
if (ret == 1)
{
ATOMIC_DEC(&((*(mgr->svc_cache)).stat.trusted_cert_cnt));
}
return ret;
}
int ssl_manager_add_crl(struct ssl_mgr* mgr, const char* pem_file)
{
return ssl_trusted_cert_storage_add(mgr->trust_CA_store, SSL_X509_OBJ_CRL, pem_file);
}
int ssl_manager_del_crl(struct ssl_mgr* mgr, const char* pem_file)
{
return ssl_trusted_cert_storage_del(mgr->trust_CA_store, SSL_X509_OBJ_CRL, pem_file);
}
void ssl_manager_reset_trust_ca(struct ssl_mgr* mgr)
{
ATOMIC_ZERO(&((*(mgr->svc_cache)).stat.trusted_cert_cnt));
ssl_trusted_cert_storage_reset(mgr->trust_CA_store);
return;
}
void ssl_manager_reset_trust_ca_finish(struct ssl_mgr* mgr)
{
ssl_trusted_cert_storage_reset_finish(mgr->trust_CA_store);
return;
}
int ssl_stream_set_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, int opt_val)
{
struct cert_verify_param *verify_param=&(upstream->up_parts.verify_param);
assert(upstream->dir==CONN_DIR_UPSTREAM);
switch(opt_type)
{
case SSL_STREAM_OPT_NO_VERIFY_SELF_SIGNED:
verify_param->no_verify_self_signed=opt_val;
break;
case SSL_STREAM_OPT_NO_VERIFY_COMMON_NAME:
verify_param->no_verify_cn=opt_val;
break;
case SSL_STREAM_OPT_NO_VERIFY_ISSUER:
verify_param->no_verify_issuer=opt_val;
break;
case SSL_STREAM_OPT_NO_VERIFY_EXPIRY_DATE:
verify_param->no_verify_expiry_date=opt_val;
break;
case SSL_STREAM_OPT_BLOCK_FAKE_CERT:
upstream->up_parts.block_fake_cert=opt_val;
break;
case SSL_STREAM_OPT_PROTOCOL_MIN_VERSION:
upstream->ssl_min_version=opt_val;
break;
case SSL_STREAM_OPT_PROTOCOL_MAX_VERSION:
upstream->ssl_max_version=opt_val;
break;
case SSL_STREAM_OPT_ENABLE_ALPN:
upstream->up_parts.apln_enabled=opt_val;
break;
case SSL_STREAM_OPT_KEYRING_FOR_TRUSTED:
upstream->up_parts.keyring_for_trusted=opt_val;
break;
case SSL_STREAM_OPT_KEYRING_FOR_UNTRUSTED:
upstream->up_parts.keyring_for_untrusted=opt_val;
break;
default:
assert(0);
return 0;
}
return 1;
}
int ssl_stream_get_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, int *opt_val)
{
struct ssl_service_status* svc=&upstream->up_parts.svc_status;
UNUSED int ret=0;
switch(opt_type)
{
case SSL_STREAM_OPT_IS_EV_CERT:
*opt_val=svc->is_ev;
break;
case SSL_STREAM_OPT_IS_CT_CERT:
*opt_val=svc->is_ct;
break;
case SSL_STREAM_OPT_IS_MUTUAL_AUTH:
*opt_val=svc->is_mutual_auth;
break;
case SSL_STREAM_OPT_PINNING_STATUS:
*opt_val=svc->cli_pinning_status;
break;
case SSL_STREAM_OPT_JA3_PINNING_STATUS:
*opt_val=svc->ja3_pinning_status;
break;
case SSL_STREAM_OPT_HAS_PROTOCOL_ERRORS:
*opt_val=svc->has_protocol_errors;
break;
default:
return -1;
}
return 0;
}
uint64_t ssl_stream_get_policy_id(struct ssl_stream *upstream)
{
uint16_t out_size;
uint64_t policy_id = 0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream);
int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_POLICY_ID, (unsigned char *)&policy_id, sizeof(policy_id), &out_size);
assert(ret == 0);
return policy_id;
}
int ssl_stream_get_decrypted_profile_id(struct ssl_stream *upstream)
{
uint16_t out_size;
int profile_id = 0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream);
int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DECRYPTION_PROFILE_ID, (unsigned char *)&profile_id, sizeof(profile_id), &out_size);
assert(ret == 0);
return profile_id;
}
int ssl_stream_get_trusted_keyring_profile_id(struct ssl_stream *upstream)
{
uint16_t out_size;
int keyring_id = 0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream);
int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_KEYRING_FOR_TRUSTED_ID, (unsigned char *)&keyring_id, sizeof(keyring_id), &out_size);
assert(ret == 0);
return keyring_id;
}
int ssl_stream_get_untrusted_keyring_profile_id(struct ssl_stream *upstream)
{
uint16_t out_size;
int keyring_id = 0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream);
int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_KEYRING_FOR_UNTRUSTED, (unsigned char *)&keyring_id, sizeof(keyring_id), &out_size);
assert(ret == 0);
return keyring_id;
}
int ssl_stream_get_string_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, char* in_buff, size_t sz)
{
const char* sni=upstream->up_parts.client_hello->sni?upstream->up_parts.client_hello->sni:"null";
switch(opt_type)
{
case SSL_STREAM_OPT_SNI:
strncpy(in_buff, sni, sz);
break;
case SSL_STREAM_OPT_ADDR:
strncpy(in_buff, upstream->tcp_stream->str_stream_info, sz);
break;
default:
assert(0);
return -1;
}
return 0;
}
Reference in New Issue View Git Blame Copy Permalink