tango-tfe/platform/src/ssl_stream.cpp

/*
 *	SSL stream implementation.
 *	Use promise as parameter of every asynchronous function call. e.g. callback functions of libevent or future-promise.
 *	Author: zhengchao@iie.ac.cn
 *	Create: 2018-8-17
 */
#include <netinet/in.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <stdint.h>
#include <sys/socket.h>

#include <event2/event.h>
#include <event2/listener.h>
#include <event2/bufferevent.h>
#include <event2/bufferevent_ssl.h>
#include <event2/buffer.h>
#include <event2/thread.h>
#include <event2/dns.h>


#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <openssl/x509.h>
#include <openssl/x509v3.h>

#include <MESA/MESA_htable.h>
#include <MESA/MESA_prof_load.h>

#include <ssl_fetch_cert.h>
#include <tfe_stream.h>
#include <tfe_utils.h>
#include <tfe_future.h>
#include <tfe_proxy.h>
#include <key_keeper.h>
#include <ssl_sess_cache.h>
#include <ssl_sess_ticket.h>
#include <ssl_trusted_cert_storage.h>
#include <ssl_utils.h>
#include <ssl_service_cache.h>
#include <platform.h>
#include <ssl_ja3.h>

static int SSL_CTX_EX_DATA_IDX_SSLMGR;
static int SSL_EX_DATA_IDX_SSLSTREAM;
static unsigned int ssl_debug;

#define MAX_NET_RETRIES        50
#define	LATENCY_WARNING_THRESHOLD_MS		1000
/*
 * Default cipher suite spec.
 * Use 'openssl ciphers -v spec' to see what ciphers are effectively enabled
 * by a cipher suite spec with a given version of OpenSSL.
 */
#define DFLT_CIPHERS "ALL:-aNULL"

static unsigned char SSL_ALPN_HTTP_1_1[]={8, 'h', 't', 't', 'p', '/', '1', '.', '1',0};
static unsigned char SSL_ALPN_HTTP_2[]={2, 'h','2',0};

/*
 * Default elliptic curve for EC cipher suites.
 */
#define DFLT_CURVE "prime256v1"

    extern long long certstore_is_unavailable;

enum ssl_stream_stat
{
	SSL_UP_NEW,
	SSL_UP_ERR,
	SSL_UP_ERR_NO_CIPHER,
	SSL_UP_ERR_UNSUPPORT_PROTO,

	SSL_UP_CLOSING,
	SSL_UP_CLOSED,
	SSL_UP_DIRTY_CLOSED,
	SSL_UP_CACHE_SZ,
	SSL_UP_CACHE_QUERY,
	SSL_UP_CACHE_HIT,

	SSL_DOWN_NEW,
	SSL_DOWN_ERR,
	SSL_DOWN_ERR_NO_CERT,
	SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK,

	SSL_DOWN_CLOSING,
	SSL_DOWN_CLOSED,
	SSL_DOWN_DIRTY_CLOSED,
	SSL_DOWN_CACHE_SZ,
	SSL_DOWN_CACHE_QUERY,
	SSL_DOWN_CACHE_HIT,
	SSL_DOWN_TICKET_NEW,
	SSL_DOWN_TICKET_REUSE,
	SSL_DOWN_TICKET_NOTFOUND,
	SSL_DOWN_TIKCET_QUERY,

	SSL_NO_CHELLO,
	SSL_NO_SNI,
	SSL_FAKE_CRT,
	KEY_KEEPER_CACHE_SIZE,
	KEY_KEEPER_ASK,
	KEY_KEEPER_ISSUE,

	SSL_SVC_PINNING,
	SSL_SVC_MAUTH,
	SSL_SVC_CT_CERT,
	SSL_SVC_EV_CERT,
	SSL_SVC_APP_NOT_PINNING,

	SSL_SVC_TRUSTED_CERT,

	SSL_STAT_MAX
};

struct ssl_mgr
{
	unsigned int sslcomp;
	unsigned int no_ssl2;
	unsigned int no_ssl3;
	unsigned int no_tls10;
	unsigned int no_tls11;
	unsigned int no_tls12;
	unsigned int no_sesscache;
	unsigned int no_sessticket;
	unsigned int no_alpn;
	unsigned int no_cert_verify;
	unsigned int no_mirror_client_cipher_suite;

	CONST_SSL_METHOD * (* sslmethod)(void);        //Parameter of SSL_CTX_new
	int ssl_min_version, ssl_max_version;
	char ssl_session_context[8];

	unsigned int sess_cache_slots;
	unsigned int sess_expire_seconds;

	unsigned int svc_cache_slots;
	unsigned int svc_expire_seconds;
	unsigned int svc_fail_as_pinning_cnt;
	unsigned int svc_fail_as_proto_err_cnt;
	unsigned int svc_succ_as_app_not_pinning_cnt;
	unsigned int svc_cnt_time_window;

	struct sess_cache * down_sess_cache;
	struct sess_cache * up_sess_cache;
	struct sess_ticket_box * down_stek_box;

	struct ssl_service_cache* svc_cache;
	ssl_stream_new_hook* on_new_upstream_cb;
	void* upstream_cb_param;
	struct sess_ticket_key ticket_key;

	char default_ciphers[TFE_SYMBOL_MAX];
	DH * dh;
	char * ecdhcurve;
	char * crl_url;

	unsigned int trusted_cert_load_local;
	struct cert_store_param cert_verify_param;
	uint8_t ssl_mode_release_buffers;
	char trusted_cert_file[TFE_PATH_MAX];
	char trusted_cert_dir[TFE_PATH_MAX];

	struct ssl_trusted_cert_storage * trust_CA_store;
	struct key_keeper * key_keeper;
	struct event_base * ev_base_gc;
    struct event * gcev;

	unsigned int log_master_key;
	char master_key_file[TFE_PATH_MAX];
	FILE* fp_master_key;

	void * logger;
	screen_stat_handle_t fs_handle;
	long long stat_val[SSL_STAT_MAX];
	int fs_id[SSL_STAT_MAX];
};

struct __ssl_stream_debug
{
	evutil_socket_t fd;
};
struct ssl_bypass_condition
{
	char bypass_ev_cert;
	char bypass_ct_cert;
	char bypass_mutual_auth;
	char bypass_cert_pinning;
};
struct ssl_upstream_parts
{

	struct cert_verify_param verify_param;
	struct cert_verify_result verify_result;
	char block_fake_cert;
	struct ssl_service_status svc_status;
	enum ssl_stream_action action;
	int apln_enabled;
	int keyring_for_trusted;
	int keyring_for_untrusted;
	struct ssl_chello * client_hello;
	uint8_t is_server_cert_verify_passed;
};
struct ssl_downstream_parts
{
	struct keyring * keyring;
};
struct ssl_stream
{
	enum tfe_conn_dir dir;
	SSL * ssl;
	struct ssl_mgr * mgr;
	union
	{
		struct ssl_upstream_parts up_parts;
		struct ssl_downstream_parts down_parts;
	};
	int ssl_min_version, ssl_max_version, negotiated_version;
	const unsigned char* alpn_selected;	//reference to SSL_ALPN_HTTP_2/SSL_ALPN_HTTP_1_1
	struct tfe_stream* tcp_stream;
	uint64_t connect_latency_ms;

	struct ssl_stream* peer;
	socklen_t peer_addrlen;
	struct sockaddr_storage peer_addr;
	struct __ssl_stream_debug _do_not_use;
	enum ssl_stream_error error;
};


struct peek_client_hello_ctx
{
	struct ssl_chello* chello;
	unsigned char sni_peek_retries;                /* max 64 SNI parse retries */
	struct event * ev;
	struct event_base * evbase;
	void * logger;
};

struct ssl_connect_server_ctx
{
	struct bufferevent * bev;
	struct ssl_stream * s_stream;
	struct ssl_mgr * mgr;
	struct sockaddr_storage addr;
	socklen_t addrlen;
	void * logger;

	evutil_socket_t fd_upstream;
	evutil_socket_t fd_downstream;

	struct tfe_stream* tcp_stream;

	struct future * f_peek_chello;
	struct timespec start,end;
};

struct ssl_connect_client_ctx
{
	struct tfe_stream* tcp_stream;
	struct ssl_stream * peer;
	X509 * origin_crt;
	int is_origin_crt_verify_passed;
	struct ssl_mgr * ssl_mgr;
	evutil_socket_t fd_downstream;

	struct future * f_ask_keyring;
	struct bufferevent * bev_down;
	struct ssl_stream * downstream;
	struct timespec start,end;
};

/*
 * SSL shutdown context.
 */
struct ssl_shutdown_ctx
{
	struct ssl_stream * s_stream;
	struct event_base * evbase;
	struct event * ev;
	struct ssl_mgr* mgr;
	enum tfe_conn_dir dir;
	unsigned int retries;
};
struct fs_spec
{
	enum ssl_stream_stat id;
	const char* name;
};

unsigned int is_ssl_debug()
{
	return ssl_debug;
}

int sslver_str2num(const char * version_str)
{
	int sslversion = -1;

	assert(OPENSSL_VERSION_NUMBER >= 0x10100000L);

	/*
	 * Support for SSLv2 and the corresponding SSLv2_method(),
	 * SSLv2_server_method() and SSLv2_client_method() functions were
	 * removed in OpenSSL 1.1.0.
	 */
	if (!strcasecmp(version_str, "ssl3")||!strcasecmp(version_str, "SSLv3")||!strcasecmp(version_str, "sslv3.0"))
	{
		sslversion = SSL3_VERSION;
	}
	else if (!strcasecmp(version_str, "tls10") || !strcasecmp(version_str, "tls1") ||
			!strcasecmp(version_str, "TLSv1.0") || !strcasecmp(version_str, "TLSv1"))
	{
		sslversion = TLS1_VERSION;
	}
	else if (!strcasecmp(version_str, "tls11")||!strcasecmp(version_str, "TLSv1.1"))
	{
		sslversion = TLS1_1_VERSION;
	}
	else if (!strcasecmp(version_str, "tls12")||!strcasecmp(version_str, "TLSv1.2"))
	{
		sslversion = TLS1_2_VERSION;
	}
	else if (!strcasecmp(version_str, "tls13")||!strcasecmp(version_str, "TLSv1.3"))
	{
		sslversion = TLS1_3_VERSION;
	}

	else
	{
		sslversion = -1;
	}

	return sslversion;
}
const char* ssl_stream_get_error_string(enum ssl_stream_error error)
{
	const char* ssl_err_str[__SSL_STREAM_R_MAX];
	ssl_err_str[SSL_STREAM_R_NO_ERROR]="OK";
	ssl_err_str[SSL_STREAM_R_SERVER_CLOSED]="server-side closed";
	ssl_err_str[SSL_STREAM_R_CLIENT_CLOSED]="client-side closed";
	ssl_err_str[SSL_STREAM_R_CONNECT_SERVER_TIMEOUT]="server-side timeout";
	ssl_err_str[SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT]="client-side timeout";
	ssl_err_str[SSL_STREAM_R_SERVER_PROTOCOL_ERROR]="server-side protocol errors";
	ssl_err_str[SSL_STREAM_R_CLIENT_PROTOCOL_ERROR]="client-side protocol errors";
	return ssl_err_str[error];
}
/*
 * Garbage collection handler.
 */
static void
ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
{
	struct ssl_mgr *mgr=(struct ssl_mgr *)arg;
	int i=0;
	if(!mgr->no_sesscache)
	{
		ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT]));
		ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT]));
	}
	struct key_keeper_stat keeper_stat = { 0 };
	key_keeper_statistic(mgr->key_keeper, &keeper_stat);
	mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times;
	mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue;
	mgr->stat_val[KEY_KEEPER_CACHE_SIZE]=keeper_stat.cached_num;

	struct ssl_service_cache_statistics svc_stat;
	memset(&svc_stat, 0, sizeof(svc_stat));
	ssl_service_cache_stat(mgr->svc_cache, &svc_stat);
	mgr->stat_val[SSL_SVC_PINNING]=svc_stat.pinning_cli_cnt;
	mgr->stat_val[SSL_SVC_MAUTH]=svc_stat.mutual_auth_cli_cnt;
	mgr->stat_val[SSL_SVC_CT_CERT]=svc_stat.ct_srv_cnt;
	mgr->stat_val[SSL_SVC_EV_CERT]=svc_stat.ev_srv_cnt;
	mgr->stat_val[SSL_SVC_APP_NOT_PINNING]=svc_stat.app_not_pinning_cnt;
	mgr->stat_val[SSL_SVC_TRUSTED_CERT]=svc_stat.trusted_cert_cnt;
	for(i=0;i<SSL_STAT_MAX;i++)
	{
		FS_operate(mgr->fs_handle, mgr->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(mgr->stat_val[i])));
	}
	if(mgr->log_master_key && mgr->fp_master_key)
	{
		fflush(mgr->fp_master_key);
	}
	return;
}
void ssl_stat_init(struct ssl_mgr * mgr)
{
	int i=0;
	const char* spec[SSL_STAT_MAX]={0};
	spec[SSL_UP_NEW]="ussl_new";
	spec[SSL_UP_ERR]="ussl_err";
	spec[SSL_UP_ERR_NO_CIPHER]="ussl_e_ciph";
	spec[SSL_UP_ERR_UNSUPPORT_PROTO]="ussl_e_prt";

	spec[SSL_UP_CLOSING]="ussl_clsing";
	spec[SSL_UP_CLOSED]="ussl_clsd";
	spec[SSL_UP_DIRTY_CLOSED]="ussl_dt_cls";
	spec[SSL_UP_CACHE_SZ]="usess_cache";
	spec[SSL_UP_CACHE_QUERY]="usess_query";
	spec[SSL_UP_CACHE_HIT]="usess_hitn";

	spec[SSL_DOWN_NEW]="dssl_new";
	spec[SSL_DOWN_ERR]="dssl_err";
	spec[SSL_DOWN_ERR_NO_CERT]="dssl_e_cert";
	spec[SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK]="dssl_e_fb";
	spec[SSL_DOWN_CLOSING]="dssl_clsing";
	spec[SSL_DOWN_CLOSED]="dssl_clsd";
	spec[SSL_DOWN_DIRTY_CLOSED]="dssl_dt_cls";
	spec[SSL_DOWN_CACHE_SZ]="dsess_cache";
	spec[SSL_DOWN_CACHE_QUERY]="dcache_query";
	spec[SSL_DOWN_CACHE_HIT]="dsess_hitn";

	if(!mgr->no_sessticket)
	{
		spec[SSL_DOWN_TICKET_NEW]="dtkt_new";
		spec[SSL_DOWN_TICKET_REUSE]="dtkt_reuse";
		spec[SSL_DOWN_TICKET_NOTFOUND]="dtkt_notfnd";
		spec[SSL_DOWN_TIKCET_QUERY]="dtkt_query";
	}
	spec[SSL_NO_CHELLO]="ssl_no_chlo";
	spec[SSL_NO_SNI]="ssl_no_sni";
	spec[SSL_FAKE_CRT]="ssl_fk_crt";
	spec[KEY_KEEPER_ASK]="kyr_ask";
	spec[KEY_KEEPER_ISSUE]="kyr_new";
	spec[KEY_KEEPER_CACHE_SIZE]="kyr_cache";

	spec[SSL_SVC_PINNING]="ssl_pinning";
	spec[SSL_SVC_MAUTH]="ssl_mauth";
	spec[SSL_SVC_CT_CERT]="ssl_ct_crt";
	spec[SSL_SVC_EV_CERT]="ssl_ev_crt";
	spec[SSL_SVC_APP_NOT_PINNING]="app_no_pinning";

	spec[SSL_SVC_TRUSTED_CERT]="trusted_cert_nums";


	for(i=0;i<SSL_STAT_MAX;i++)
	{
		if(spec[i]!=NULL)
		{
			mgr->fs_id[i]=FS_register(mgr->fs_handle, FS_STYLE_FIELD, FS_CALC_CURRENT,spec[i]);
		}
	}

	int	value=mgr->fs_id[SSL_UP_CACHE_HIT];
	FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
	value=mgr->fs_id[SSL_UP_CACHE_QUERY];
	FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));

	FS_register_ratio(mgr->fs_handle,
					mgr->fs_id[SSL_UP_CACHE_HIT],
					mgr->fs_id[SSL_UP_CACHE_QUERY],
					1,
					FS_STYLE_STATUS,
					FS_CALC_CURRENT,
					"usess_hit");

	value=mgr->fs_id[SSL_DOWN_CACHE_HIT];
	FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));
	value=mgr->fs_id[SSL_DOWN_CACHE_QUERY];
	FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));

	FS_register_ratio(mgr->fs_handle,
					mgr->fs_id[SSL_DOWN_CACHE_HIT],
					mgr->fs_id[SSL_DOWN_CACHE_QUERY],
					1,
					FS_STYLE_STATUS,
					FS_CALC_CURRENT,
					"dsess_hit");

	if(!mgr->no_sessticket)
	{
		value=mgr->fs_id[SSL_DOWN_TIKCET_QUERY];
		FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));

		value=mgr->fs_id[SSL_DOWN_TICKET_REUSE];
		FS_set_para(mgr->fs_handle, ID_INVISBLE, &value, sizeof(value));

		FS_register_ratio(mgr->fs_handle,
						mgr->fs_id[SSL_DOWN_TICKET_REUSE],
						mgr->fs_id[SSL_DOWN_TIKCET_QUERY],
						1,
						FS_STYLE_STATUS,
						FS_CALC_CURRENT,
						"dtkt_hit");
	}

	struct timeval gc_delay = {0, 500*1000};	//Microseconds, we set 500 miliseconds here.
	mgr->gcev = event_new(mgr->ev_base_gc, -1, EV_PERSIST, ssl_stream_gc_cb, mgr);
    evtimer_add(mgr->gcev, &gc_delay);

	return;
}
static void downstream_ossl_init(struct ssl_stream* s_stream);
static int upstream_ossl_init(struct ssl_stream* s_stream);

static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr);

struct ssl_chello * ssl_peek_result_release_chello(future_result_t * result)
{
	struct peek_client_hello_ctx* ctx= (struct peek_client_hello_ctx*) result;
	struct ssl_chello * p = ctx->chello;
	ctx->chello=NULL;
	return p;
}

struct ssl_stream * ssl_stream_new(struct ssl_mgr * mgr, evutil_socket_t fd, enum tfe_conn_dir dir,
	struct ssl_chello * client_hello, struct keyring * kyr, struct ssl_stream * peer, struct tfe_stream* tcp_stream)
{

	UNUSED int ret = 0;

	struct ssl_stream * s_stream = ALLOC(struct ssl_stream, 1);
	s_stream->dir = dir;
	s_stream->mgr = mgr;
	s_stream->_do_not_use.fd = fd;
	s_stream->ssl_max_version=mgr->ssl_max_version;
	s_stream->ssl_min_version=mgr->ssl_min_version;
	s_stream->peer=peer;
	s_stream->tcp_stream=tcp_stream;
	ret = getpeername(fd, (struct sockaddr *) (&s_stream->peer_addr), &(s_stream->peer_addrlen));
	switch (dir)
	{
		case CONN_DIR_DOWNSTREAM:
			assert(peer!=NULL);
			ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_NEW]));
			s_stream->down_parts.keyring = kyr;
			s_stream->peer=peer;
			break;
		case CONN_DIR_UPSTREAM:
			ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_NEW]));
			s_stream->up_parts.verify_param.no_verify_cn=1;
			s_stream->up_parts.client_hello = client_hello;
			s_stream->ssl_min_version=client_hello->min_version.ossl_format;
			s_stream->ssl_max_version=client_hello->max_version.ossl_format;
			break;
		default: assert(0);
	}
	return s_stream;
}
static void ssl_stream_free(struct ssl_stream * s_stream)
{
	if(s_stream->ssl)
	{
		SSL_free(s_stream->ssl);
		s_stream->ssl = NULL;
	}
	switch (s_stream->dir)
	{
		case CONN_DIR_DOWNSTREAM:
			if (s_stream->down_parts.keyring != NULL)
			{
				key_keeper_free_keyring(s_stream->down_parts.keyring);
				s_stream->down_parts.keyring = NULL;
			}
			ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_DOWN_CLOSED]));
			break;
		case CONN_DIR_UPSTREAM:
			if (s_stream->up_parts.client_hello != NULL)
			{
				ssl_chello_free(s_stream->up_parts.client_hello);
				s_stream->up_parts.client_hello = NULL;
			}
			ATOMIC_INC(&(s_stream->mgr->stat_val[SSL_UP_CLOSED]));
			break;
		default: assert(0);
	}
	if(s_stream->alpn_selected)
	{
		s_stream->alpn_selected=NULL;
	}
	s_stream->mgr = NULL;
	free(s_stream);
	return;
}


static void log_ssl_master_key(struct ssl_stream *s_stream, tfe_conn_dir dir, FILE* fp)
{
	char* key_str=NULL;
	key_str=ssl_ssl_masterkey_to_str(s_stream->ssl);
	char time_str[TFE_SYMBOL_MAX];
	time_t now=time(NULL);
	tfe_thread_safe_ctime(&now, time_str, sizeof(time_str));

	fprintf(fp, "#%s %s %s\n%s\n", time_str, tfe_stream_conn_dir_to_str(dir), s_stream->tcp_stream->str_stream_info ? s_stream->tcp_stream->str_stream_info : NULL, key_str);

	free(key_str);
	return;
}

void ssl_manager_destroy(struct ssl_mgr * mgr)
{
	if (mgr->key_keeper)
	{
		key_keeper_destroy(mgr->key_keeper);
	}
	if (mgr->trust_CA_store)
	{
		ssl_trusted_cert_storage_destroy(mgr->trust_CA_store);
		mgr->trust_CA_store = NULL;
	}
	if(mgr->down_sess_cache)
	{
		ssl_sess_cache_destroy(mgr->down_sess_cache);
	}
	if(mgr->up_sess_cache)
	{
		ssl_sess_cache_destroy(mgr->up_sess_cache);
	}
	if(mgr->gcev)
	{
		 event_free(mgr->gcev);
	}
	if(mgr->fp_master_key)
	{
		fclose(mgr->fp_master_key);
	}
	free(mgr);
}


struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section,
	struct event_base * ev_base_gc, struct key_keeper * key_keeper, void * logger)
{
    unsigned int stek_group_num = 0;
    unsigned int stek_rotation_time = 0;

	struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1);
	int ret = 0;

    char ja3_table_name[TFE_STRING_MAX] = {0};
    char version_str[TFE_SYMBOL_MAX] = {};
	mgr->logger = logger;
	mgr->ev_base_gc=ev_base_gc;

	if (ssl_mid_cert_kafka_logger_create(ini_profile, section))
	{
		goto error_out;
	}

    MESA_load_profile_uint_def(ini_profile, section, "ssl_debug", &(ssl_debug), 0);
    MESA_load_profile_string_def(ini_profile, section, "ssl_min_version", version_str, sizeof(version_str), "ssl3");
	mgr->ssl_min_version = sslver_str2num(version_str);

	if (mgr->ssl_min_version < 0)
	{
		TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str);
		goto error_out;
	}

	MESA_load_profile_string_def(ini_profile, section, "ssl_max_version", version_str, sizeof(version_str), "tls12");
	mgr->ssl_max_version = sslver_str2num(version_str);

	if (mgr->ssl_max_version < 0)
	{
		TFE_LOG_ERROR(logger, "Unsupported SSL/TLS protocol %s", version_str);
		goto error_out;
	}

	ret=ssl_init();
	if(ret<0)
	{
		TFE_LOG_ERROR(logger, "OpenSSL global init failed.");
		goto error_out;
	}
	//tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0.
	mgr->sslmethod = TLS_method;
    SSL_CTX_EX_DATA_IDX_SSLMGR = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL);
	SSL_EX_DATA_IDX_SSLSTREAM  = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL);
	MESA_load_profile_uint_def(ini_profile, section, "ssl_compression", &(mgr->sslcomp), 1);
	MESA_load_profile_uint_def(ini_profile, section, "no_ssl2", &(mgr->no_ssl2), 1);
	MESA_load_profile_uint_def(ini_profile, section, "no_ssl3", &(mgr->no_ssl3), 0);
	MESA_load_profile_uint_def(ini_profile, section, "no_tls10", &(mgr->no_tls10), 0);
	MESA_load_profile_uint_def(ini_profile, section, "no_tls11", &(mgr->no_tls11), 0);
	MESA_load_profile_uint_def(ini_profile, section, "no_tls12", &(mgr->no_tls12), 0);
	MESA_load_profile_string_def(ini_profile, section, "default_ciphers", mgr->default_ciphers,
		sizeof(mgr->default_ciphers), DFLT_CIPHERS);

	MESA_load_profile_uint_def(ini_profile, section, "no_session_cache", &(mgr->no_sesscache), 0);
	MESA_load_profile_uint_def(ini_profile, section, "no_session_ticket", &(mgr->no_sessticket), 0);
	MESA_load_profile_uint_def(ini_profile, section, "no_alpn", &(mgr->no_alpn), 0);
	MESA_load_profile_uint_def(ini_profile, section, "no_cert_verify", &(mgr->no_cert_verify), 0);

	MESA_load_profile_uint_def(ini_profile, section, "no_mirror_client_cipher_suite",
		&(mgr->no_mirror_client_cipher_suite), 0);
	MESA_load_profile_uint_def(ini_profile, section, "session_cache_slots",
		&(mgr->sess_cache_slots), 4 * 1024 * 1024);
	MESA_load_profile_uint_def(ini_profile, section, "session_cache_expire_seconds",
		&(mgr->sess_expire_seconds), 30 * 60);


	if(!mgr->no_sesscache)
	{
		mgr->up_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM);
		mgr->down_sess_cache = ssl_sess_cache_create(mgr->sess_cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM);
	}

    MESA_load_profile_uint_def(ini_profile, section, "stek_group_num", &stek_group_num, 1);
    MESA_load_profile_uint_def(ini_profile, section, "stek_rotation_time", &stek_rotation_time, 3600);

	if(!mgr->no_sessticket)
    {
        mgr->down_stek_box = sess_ticket_box_create(ev_base_gc, stek_group_num, stek_rotation_time, logger);
    }
	MESA_load_profile_uint_def(ini_profile, section, "service_cache_slots",
		&(mgr->svc_cache_slots), 4 * 1024 * 1024);
	MESA_load_profile_uint_def(ini_profile, section, "service_cache_expire_seconds",
		&(mgr->svc_expire_seconds), 5 * 60);
	MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_as_pinning_cnt",
		&(mgr->svc_fail_as_pinning_cnt), 4);
	MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_as_proto_err_cnt",
		&(mgr->svc_fail_as_proto_err_cnt), 5);
	MESA_load_profile_uint_def(ini_profile, section, "service_cache_fail_time_window",
		&(mgr->svc_cnt_time_window), 30);

    MESA_load_profile_string_def(ini_profile, section, "ssl_ja3_table", ja3_table_name, sizeof(ja3_table_name), "PXY_SSL_FINGERPRINT");
    mgr->svc_cache=ssl_service_cache_create(mgr->svc_cache_slots, mgr->svc_expire_seconds,
											mgr->svc_fail_as_pinning_cnt,
											mgr->svc_fail_as_proto_err_cnt,
											mgr->svc_cnt_time_window, ja3_table_name);
	if (mgr->svc_cache == NULL)
	{
        TFE_LOG_ERROR(logger, "Failed to create service cache");
        goto error_out;
    }

    mgr->key_keeper = key_keeper;
	MESA_load_profile_uint_def(ini_profile, section, "trusted_cert_load_local",
		&(mgr->trusted_cert_load_local), 1);

	MESA_load_profile_uint_def(ini_profile, section, "check_cert_crl", &(mgr->cert_verify_param.check_crl), 0);
	if(mgr->trusted_cert_load_local)//Other wise, use policy defined trusted CA file.
	{
		MESA_load_profile_string_def(ini_profile, section, "trusted_cert_file", mgr->trusted_cert_file, sizeof(mgr->trusted_cert_file),
			"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem");

		MESA_load_profile_string_def(ini_profile, section, "trusted_cert_dir", mgr->trusted_cert_dir, sizeof(mgr->trusted_cert_dir),
			"./resource/tfe/trusted_storage");
	}
	mgr->trust_CA_store = ssl_trusted_cert_storage_create(mgr->trusted_cert_file, mgr->trusted_cert_dir, &(mgr->cert_verify_param));
	if (mgr->trust_CA_store == NULL)
	{
		TFE_LOG_ERROR(logger, "Failed at creating X509_STORE");
		goto error_out;
	}

	memcpy(mgr->ssl_session_context, "mesa-tfe", sizeof(mgr->ssl_session_context));

	MESA_load_profile_uint_def(ini_profile, section, "log_master_key", &(mgr->log_master_key), 0);
	MESA_load_profile_string_def(ini_profile, section, "key_log_file", mgr->master_key_file, sizeof(mgr->master_key_file),
		"./sslkeylog.log");
	if(mgr->log_master_key)
	{
		mgr->fp_master_key=fopen(mgr->master_key_file, "a");
		if(mgr->fp_master_key==NULL)
		{
			TFE_LOG_ERROR(logger, "Failed at open master key log file %s", mgr->master_key_file);
			mgr->log_master_key=0;
		}
	}

	mgr->fs_handle=tfe_proxy_get_fs_handle();
	ssl_stat_init(mgr);
	return mgr;

error_out:
	ssl_manager_destroy(mgr);
	return NULL;
}
void ssl_manager_set_new_upstream_cb(struct ssl_mgr * mgr, ssl_stream_new_hook* new_upstream_cb, void* u_para)
{
	mgr->on_new_upstream_cb=new_upstream_cb;
	mgr->upstream_cb_param=u_para;
	return;
}

void peek_client_hello_ctx_free(struct peek_client_hello_ctx * _ctx)
{
	event_free(_ctx->ev);
	_ctx->ev = NULL;
	if(_ctx->chello!=NULL)
	{
		ssl_chello_free(_ctx->chello);
		_ctx->chello=NULL;
	}
	free(_ctx);
}

void peek_client_hello_ctx_free_cb(void * p)
{
	struct peek_client_hello_ctx * _ctx = (struct peek_client_hello_ctx *)p;
	return peek_client_hello_ctx_free(_ctx);
}

static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg)
{
	struct promise * promise = (struct promise *) arg;
	//use promise_get_ctx instead of promise_dettach_ctx for try more times.
	struct peek_client_hello_ctx * ctx = (struct peek_client_hello_ctx *) promise_get_ctx(promise);
	enum chello_parse_result chello_status=CHELLO_PARSE_INVALID_FORMAT;
	struct ssl_chello* chello=NULL;
	const char * reason = NULL;
	unsigned char buf[2048];
	ssize_t n = 0;

	n = recv(fd, buf, sizeof(buf), MSG_PEEK);
	if (n == -1)
	{
	    char* addr_string=tfe_string_addr_create_by_fd(fd, CONN_DIR_DOWNSTREAM);
        TFE_LOG_ERROR(ctx->logger, "Error peeking on fd, aborting connection, addr_string is %s, errno is %d, errmsg is %s\n", addr_string, errno, strerror(errno));
		/* after log, reset errno */
		errno = 0;
        free(addr_string);
		goto failed;
	}

	if (n == 0)
	{
		goto failed;
	}

	chello=ssl_chello_parse(buf, n, &chello_status);
	switch(chello_status)
	{
		case CHELLO_PARSE_SUCCESS:
		{
            if (is_ssl_debug())
            {
                char *addr = tfe_string_addr_create_by_fd(fd, CONN_DIR_DOWNSTREAM);
                struct ssl_ja3 *fingerprint = ssl_ja3_generate_fingerprint(buf, n);
                TFE_LOG_INFO(ctx->logger, "ja3:%s, hash:%s addr:%s\n", fingerprint ? fingerprint->ja3 : "null", fingerprint ? fingerprint->ja3_hash : "null", addr);
                ssl_ja3_free(fingerprint);
                free(addr);
			}
			
            promise_dettach_ctx(promise);
			ctx->chello=chello;
			promise_success(promise, ctx);
			peek_client_hello_ctx_free(ctx);
			break;
		}
		case CHELLO_PARSE_NOT_ENOUGH_BUFF:
		{
			ssl_chello_free(chello);
			chello=NULL;
			if (ctx->sni_peek_retries++ > MAX_NET_RETRIES)
			{
				TFE_LOG_ERROR(ctx->logger, "Peek failed due to too many retries\n");
				reason = "too many peek retries";
				goto failed;
			}

			/* ssl_tls_clienthello_parse indicates that we
			* should retry later when we have more data, and we
			* haven't reached the maximum retry count yet.
			* Reschedule this event as timeout-only event in
			* order to prevent busy looping over the read event.
			* Because we only peeked at the pending bytes and
			* never actually read them, fd is still ready for
			* reading now.	We use 25 * 0.2 s = 5 s timeout. */
			struct timeval retry_delay = {0, 100};
		    event_del(ctx->ev);
			event_free(ctx->ev);
			ctx->ev = event_new(ctx->evbase, fd, 0, peek_client_hello_cb, promise);
			assert(ctx->ev != NULL);
			event_add(ctx->ev, &retry_delay);
			break;
		}
		case CHELLO_PARSE_INVALID_FORMAT:
		{
			ssl_chello_free(chello);
			chello=NULL;
			TFE_LOG_ERROR(ctx->logger,
				"Peeking did not yield a (truncated) ClientHello message, aborting connection\n");
			reason = "see no client hello";
			goto failed;
			break;
		}
		default:
			assert(0);
	}
	return;

failed:
	promise_dettach_ctx(promise);
	promise_failed(promise, FUTURE_ERROR_EXCEPTION, reason);
	peek_client_hello_ctx_free(ctx);
	return;
}

static void ssl_async_peek_client_hello(struct future * f, evutil_socket_t fd, struct event_base * evbase,
	void * logger)
{
	struct promise * p = future_to_promise(f);
	struct peek_client_hello_ctx * ctx = ALLOC(struct peek_client_hello_ctx, 1);
	ctx->ev = event_new(evbase, fd, EV_READ, peek_client_hello_cb, p);
	ctx->evbase = evbase;
	ctx->logger = logger;
	promise_set_ctx(p, (void *) ctx, peek_client_hello_ctx_free_cb);
	event_add(ctx->ev, NULL);
	return;
}
int ossl_client_cert_cb(SSL *ssl, X509 **x509, EVP_PKEY **pkey)
{
	struct ssl_stream* s_upstream=NULL;
	s_upstream=(struct ssl_stream*)SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM);
	s_upstream->up_parts.svc_status.is_mutual_auth=1;
	ssl_service_cache_write(s_upstream->mgr->svc_cache, s_upstream->up_parts.client_hello, s_upstream->tcp_stream, &s_upstream->up_parts.svc_status);
	return 0;
}

/*
 * Create new SSL context for outgoing connections to the original destination.
 * If hostname sni is provided, use it for Server Name Indication.
 */
static int upstream_ossl_init(struct ssl_stream* s_stream)
{
	SSL_CTX * sslctx = NULL;
	SSL * ssl = NULL;
	SSL_SESSION * sess = NULL;
	struct ssl_mgr * mgr = s_stream->mgr;
	struct ssl_chello * chello=s_stream->up_parts.client_hello;
	sslctx = SSL_CTX_new(mgr->sslmethod());
	if (sslctx == NULL)
	{
		TFE_LOG_ERROR(mgr->logger, "ssl ctx new failed.");
		return -1;
	}
	sslctx_set_opts(sslctx, mgr);
	int ret=0;
	char common_cipher[TFE_STRING_MAX]={0}, tls13_cipher[TFE_STRING_MAX]={0};
	if(chello->cipher_suites)
	{
		ssl_cipher_suites_to_name(chello->cipher_suites, chello->cipher_suites_len,
			common_cipher, sizeof(common_cipher), tls13_cipher, sizeof(tls13_cipher));
	}
	if(strlen(common_cipher)>0)
	{
		//SSL_CTX_set_cipher_list() and SSL_set_cipher_list() return 1 if any cipher could be selected and 0 on complete failure.
		ret=SSL_CTX_set_cipher_list(sslctx, common_cipher);
		if(ret==0)
		{
			TFE_LOG_ERROR(mgr->logger, "ssl ctx set cipher list %s failed.", common_cipher);
			SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers);
		}
	}
	else
	{
		ret=SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers);
	}
	if(strlen(tls13_cipher)>0 && s_stream->ssl_max_version==TLS1_3_VERSION)
	{
		//SSL_CTX_set_ciphersuites(sslctx, tls13_cipher);

	}
	if (SSL_CTX_set_min_proto_version(sslctx, s_stream->ssl_min_version) == 0)
	{
		SSL_CTX_free(sslctx);
		TFE_LOG_ERROR(mgr->logger, "ssl set min proto version failed %d.", s_stream->ssl_min_version);
		return -1;
	}
	if (SSL_CTX_set_max_proto_version(sslctx, s_stream->ssl_max_version) == 0)
	{
		SSL_CTX_free(sslctx);
		TFE_LOG_ERROR(mgr->logger, "ssl set max proto version failed %d.", s_stream->ssl_max_version);
		return -1;
	}

	SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL);
	SSL_CTX_set_client_cert_cb(sslctx, ossl_client_cert_cb);

	if(mgr->no_sesscache)
	{
		SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_OFF);
	}
	else
	{
		SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_NO_INTERNAL);
		/* session resuming based on remote endpoint address and port */
		sess = up_session_get(mgr->up_sess_cache,
							(struct sockaddr *) &(s_stream->peer_addr), s_stream->peer_addrlen, chello->sni,
							s_stream->ssl_min_version, s_stream->ssl_max_version);
		if (sess)
		{
			ret=SSL_CTX_add_session(sslctx, sess);    /* increments sess refcount */
			assert(ret==1);
			SSL_SESSION_free(sess);
		}
	}
	ssl = SSL_new(sslctx);
	SSL_CTX_free(sslctx);  /* SSL_new() increments refcount */
	if (!ssl)
	{
		TFE_LOG_ERROR(mgr->logger, "ssl new failed.");
		return -1;
	}
	SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream);

	if (chello->sni)
	{
		SSL_set_tlsext_host_name(ssl, chello->sni);
	}
	if (chello->alpn && s_stream->up_parts.apln_enabled)
	{
		ret=SSL_set_alpn_protos(ssl, (unsigned char*)chello->alpn, strlen(chello->alpn));
		assert(ret==0);
	}
	/* lower memory footprint for idle connections */
	SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS);

	s_stream->ssl=ssl;
	return 0;
}

void ssl_connect_server_ctx_free(struct ssl_connect_server_ctx * ctx)
{
	if (ctx->s_stream != NULL)
	{
		ssl_stream_free(ctx->s_stream);
	}

	if (ctx->bev != NULL)
	{
		bufferevent_free(ctx->bev);
		ctx->bev = NULL;
	}

	if (ctx->f_peek_chello != NULL)
	{
		future_destroy(ctx->f_peek_chello);
		ctx->f_peek_chello = NULL;
	}

	free(ctx);
	return;
}

void wrap_ssl_connect_server_ctx_free(void *p)
{
	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *)p;
	ssl_connect_server_ctx_free(ctx);
}
enum ssl_stream_action ssl_upstream_create_result_release_action(future_result_t * result)
{
	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
	return ctx->s_stream->up_parts.action;
}
evutil_socket_t ssl_upstream_create_result_release_fd(future_result_t * result)
{
	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
	return ctx->fd_upstream;
}

struct ssl_stream * ssl_upstream_create_result_release_stream(future_result_t * result)
{
	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
	struct ssl_stream * ret = ctx->s_stream;
	ctx->s_stream = NULL;                            //giveup ownership
	return ret;
}

struct bufferevent * ssl_upstream_create_result_release_bev(future_result_t * result)
{
	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) result;
	struct bufferevent * ret = ctx->bev;
	ctx->bev = NULL;                                //giveup ownership
	return ret;
}
const char* ssl_stream_dump_info(struct ssl_stream *stream, char* buffer, size_t sz)
{
	snprintf(buffer, sz, "%s:%s", SSL_get_version(stream->ssl),
								stream->dir==CONN_DIR_UPSTREAM ? stream->up_parts.client_hello->sni:NULL);
	return buffer;
}
void ssl_stream_set_cmsg_string(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, const char* value_str)
{
	struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream);
	UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)value_str, (uint16_t)strlen(value_str));
	assert(ret==0);
	return;
}
static void ssl_stream_set_cmsg_uint8(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, uint8_t value_int)
{
	struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream);
	UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)&value_int, (uint16_t)sizeof(value_int));
	assert(ret==0);
	return;
}
static void ssl_stream_set_cmsg_integer(struct ssl_stream* stream, enum tfe_cmsg_tlv_type type, uint64_t value_int)
{
	struct tfe_cmsg* cmsg=tfe_stream_get0_cmsg(stream->tcp_stream);
	UNUSED int ret=tfe_cmsg_set(cmsg, type, (const unsigned char*)&value_int, (uint16_t)sizeof(value_int));
	assert(ret==0);
	return;
}

unsigned long ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struct ssl_mgr* mgr)
{
	unsigned long sslerr=0, ret_sslerr=0;
	int fd=bufferevent_getfd(bev);
	char* addr_string=tfe_string_addr_create_by_fd(fd, dir);
	void* logger=mgr->logger;
	int fs_id=-1;

	/* Can happen for socket errs, ssl errs;
	 * may happen for unclean ssl socket shutdowns. */
	sslerr = bufferevent_get_openssl_error(bev);
	ret_sslerr=ERR_GET_REASON(sslerr);
	switch(ret_sslerr)
	{
		case SSL_R_INAPPROPRIATE_FALLBACK:
			if(dir==CONN_DIR_DOWNSTREAM)	fs_id=SSL_DOWN_ERR_INAPPROPRIATE_FALLBACK;
			break;
		case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN:
			if(dir==CONN_DIR_DOWNSTREAM)	fs_id=SSL_DOWN_ERR_NO_CERT;
			break;
		case SSL_R_UNSUPPORTED_COMPRESSION_ALGORITHM:
		case SSL_R_UNSUPPORTED_PROTOCOL:
		case SSL_R_UNSUPPORTED_SSL_VERSION:
		case SSL_R_UNSUPPORTED_STATUS_TYPE:
		case SSL_R_NO_PROTOCOLS_AVAILABLE:
			if(dir==CONN_DIR_UPSTREAM)	fs_id=SSL_UP_ERR_UNSUPPORT_PROTO;
		case SSL_R_NO_CIPHERS_AVAILABLE:
			if(dir==CONN_DIR_UPSTREAM) 	fs_id=SSL_UP_ERR_NO_CIPHER;
			break;
		default:
			fs_id=-1;
	}
	if(fs_id>=0)
	{
		mgr->stat_val[fs_id]++;
	}
	if (!errno && !sslerr)
	{
		/* We have disabled notification for unclean shutdowns
		 * so this should not happen; log a warning. */
		TFE_LOG_ERROR(logger,"Warning: Spurious error from "
					   "bufferevent (errno=0, sslerr=0)\n");
	}
	else if (ERR_GET_REASON(sslerr) ==
		SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE)
	{
		/* these can happen due to client cert auth,
		 * only log error if debugging is activated */
		TFE_LOG_ERROR(logger,"Handshake Error from bufferevent of ssl %s %s: "
					   "%i:%s %lu:%i:%s:%i:%s:%i:%s",
			tfe_stream_conn_dir_to_str(dir),
			addr_string,
			errno,
			errno ? strerror(errno) : "-",
			sslerr,
			ERR_GET_REASON(sslerr),
			sslerr ?
			ERR_reason_error_string(sslerr) : "-",
			ERR_GET_LIB(sslerr),
			sslerr ?
			ERR_lib_error_string(sslerr) : "-",
			ERR_GET_FUNC(sslerr),
			sslerr ?
			ERR_func_error_string(sslerr) : "-");
			/* after log, reset errno */
			errno = 0;
		while ((sslerr = bufferevent_get_openssl_error(bev)))
		{
			TFE_LOG_ERROR(logger,"Additional SSL error: "
						   "%lu:%i:%s:%i:%s:%i:%s",
				sslerr,
				ERR_GET_REASON(sslerr),
				ERR_reason_error_string(sslerr),
				ERR_GET_LIB(sslerr),
				ERR_lib_error_string(sslerr),
				ERR_GET_FUNC(sslerr),
				ERR_func_error_string(sslerr));
		}
	}
	else
	{
		/* real errors */
		TFE_LOG_ERROR(logger,"Error from bufferevent of ssl %s %s: "
					   "%i:%s %lu:%i:%s:%i:%s:%i:%s",
			tfe_stream_conn_dir_to_str(dir),
			addr_string,
			errno,
			errno ? strerror(errno) : "-",
			sslerr,
			ERR_GET_REASON(sslerr),
			sslerr ?
			ERR_reason_error_string(sslerr) : "-",
			ERR_GET_LIB(sslerr),
			sslerr ?
			ERR_lib_error_string(sslerr) : "-",
			ERR_GET_FUNC(sslerr),
			sslerr ?
			ERR_func_error_string(sslerr) : "-");
			/* after log, reset errno */
			errno = 0;
		while ((sslerr = bufferevent_get_openssl_error(bev)))
		{
			TFE_LOG_ERROR(logger,"Additional SSL error: "
						   "%lu:%i:%s:%i:%s:%i:%s\n",
				sslerr,
				ERR_GET_REASON(sslerr),
				ERR_reason_error_string(sslerr),
				ERR_GET_LIB(sslerr),
				ERR_lib_error_string(sslerr),
				ERR_GET_FUNC(sslerr),
				ERR_func_error_string(sslerr));
		}
	}
	free(addr_string);
	return ret_sslerr;
}
void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr, struct ssl_mgr* mgr)
{
	struct ssl_upstream_parts* s_upstream=NULL;
	assert(sslerr);
	switch(s_stream->dir)
	{
		case	CONN_DIR_DOWNSTREAM:
			 s_upstream= &(s_stream->peer->up_parts);
			if( (sslerr==SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN ||
				 sslerr==SSL_R_SSLV3_ALERT_BAD_CERTIFICATE ||
				 sslerr==SSL_R_TLSV1_ALERT_UNKNOWN_CA ||
				 sslerr==SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE ||
				 sslerr==SSL_R_UNKNOWN_CERTIFICATE_TYPE)
				&& s_upstream->is_server_cert_verify_passed
				&& s_upstream->verify_result.is_hostmatched)
			{
				s_upstream->svc_status.cli_pinning_status=PINNING_ST_PINNING;

				// This feature is also displayed when the root certificate is not installed.
				// Here no longer set the pinning state to cmsg, but use the pinng state
				// corrected by app_is_not_pinnig in the peek_chello_on_succ function
				// ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING);
				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status);
			}
			else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN)
			{
				s_upstream->svc_status.has_protocol_errors=1;
				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status);
			}
			break;
		case CONN_DIR_UPSTREAM:
			s_upstream=&(s_stream->up_parts);
			s_upstream->svc_status.has_protocol_errors=1;
			ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->up_parts.svc_status));
			break;
		default:
			assert(0);
	}
	return;
}

void ssl_stream_process_zero_eof(struct ssl_stream * s_stream, struct ssl_mgr* mgr)
{
	struct ssl_upstream_parts* s_upstream=NULL;
	if(s_stream->dir==CONN_DIR_UPSTREAM)
	{
		return;
	}
	assert(mgr==s_stream->mgr);
	s_upstream=&s_stream->peer->up_parts;
	if(s_upstream->verify_result.is_hostmatched && s_upstream->is_server_cert_verify_passed )
	{
		const char *sni = s_upstream->client_hello ? (s_upstream->client_hello->sni ? s_upstream->client_hello->sni : "null"): "null";
		TFE_LOG_DEBUG(mgr->logger, "sni:%s cert verify passed and hit zero eof, set protocol errors", sni);
		s_upstream->svc_status.has_protocol_errors=1;
		ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->peer->up_parts.svc_status));
	}
	s_stream->error=SSL_STREAM_R_CLIENT_CLOSED;
	return;
}


/*
 * Callback for meta events on the up- and downstream connection bufferevents.
 * Called when EOF has been reached, a connection has been made, and on errors.
 */
static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, void * arg)
{
	struct promise * p = (struct promise *) arg;
	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p);

	struct ssl_stream * s_stream = ctx->s_stream;
	assert(s_stream->dir==CONN_DIR_UPSTREAM);
	struct ssl_upstream_parts* s_upstream=&(s_stream->up_parts);
	struct ssl_mgr* mgr=s_stream->mgr;
	SSL_SESSION * ssl_sess = NULL;
	char error_str[TFE_STRING_MAX];
	uint64_t jiffies_ms;
	unsigned long sslerr=0;
	if (events & BEV_EVENT_ERROR)
	{
		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
		sslerr=ssl_stream_log_error(bev, CONN_DIR_UPSTREAM, ctx->mgr);
		if(sslerr)
		{
			ssl_stream_process_error(s_stream, sslerr, mgr);
		}
		s_stream->error=SSL_STREAM_R_SERVER_PROTOCOL_ERROR;
	}
	else if(events & BEV_EVENT_EOF)
	{
		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
		s_stream->error=SSL_STREAM_R_SERVER_CLOSED;
	}
	else if(events & BEV_EVENT_TIMEOUT)
	{
		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_ERR]));
		s_stream->error=SSL_STREAM_R_CONNECT_SERVER_TIMEOUT;
	}
	else if(events & BEV_EVENT_CONNECTED)
	{
		bufferevent_disable(ctx->bev, EV_READ | EV_WRITE);
		bufferevent_setcb(ctx->bev, NULL, NULL, NULL, NULL); //leave a clean bev for on_success
		clock_gettime(CLOCK_MONOTONIC, &(ctx->end));
		jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000;
		if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS)
		{
			TFE_LOG_INFO(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s",
									jiffies_ms,
									s_stream->tcp_stream->str_stream_info,
									s_upstream->client_hello->sni);
		}
		s_stream->connect_latency_ms=jiffies_ms;
		ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_SERVER_SIDE_LATENCY, jiffies_ms);

		if(!SSL_session_reused(s_stream->ssl))
		{
			if(mgr->no_cert_verify)
			{
				s_upstream->is_server_cert_verify_passed=1;
			}
			else
			{
				s_upstream->is_server_cert_verify_passed = !!ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store,
													s_stream->ssl, s_stream->up_parts.client_hello->sni, &(s_stream->up_parts.verify_param),
													error_str, sizeof(error_str), &(s_stream->up_parts.verify_result));
				TFE_LOG_DEBUG(g_default_logger,
					"addr:%s, sni:%s, is_cert_verify_passed:%d, cet_real_untrust:%d, verify_host_fail:%d, verify_issure_fail:%d, verify_self_signed_fail:%d, verify_expiry_date_fail:%d, verify_other_fail:%d, %s",
					s_stream->tcp_stream->str_stream_info,
					s_upstream->client_hello->sni,
					s_upstream->is_server_cert_verify_passed,
					((s_upstream->verify_param.real_untrust & 0xff) ? 1 : 0),
					((s_upstream->verify_param.real_untrust & 0x01) ? 1 : 0),
					((s_upstream->verify_param.real_untrust & 0x02) ? 1 : 0),
					((s_upstream->verify_param.real_untrust & 0x04) ? 1 : 0),
					((s_upstream->verify_param.real_untrust & 0x08) ? 1 : 0),
					((s_upstream->verify_param.real_untrust & 0x10) ? 1 : 0),
					(s_upstream->is_server_cert_verify_passed == 0 ? error_str : ""));

				s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct;
				s_upstream->svc_status.is_ev=s_upstream->verify_result.is_ev;
				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &(s_upstream->svc_status));
				TFE_LOG_DEBUG(mgr->logger, "stream:%s sni:%s hostmatch:%d, ct:%d, ev:%d",
											s_stream->tcp_stream->str_stream_info,
											s_upstream->client_hello->sni,
											s_upstream->verify_result.is_hostmatched,
											s_upstream->verify_result.is_ct,
											s_upstream->verify_result.is_ev);
				if((!s_upstream->is_server_cert_verify_passed || !s_upstream->verify_result.is_hostmatched) && s_upstream->block_fake_cert)
				{
					s_stream->up_parts.action=SSL_ACTION_SHUTDOWN;
				}
			}
			if(s_upstream->is_server_cert_verify_passed)
			{
				if(!mgr->no_sesscache && s_stream->up_parts.action==SSL_ACTION_INTERCEPT)
				{
					//ONLY verified session is cacheable.
					//The reference count of the SSL_SESSION is not incremented, so no need to free.
					ssl_sess = SSL_get0_session(s_stream->ssl);
					up_session_set(mgr->up_sess_cache, (struct sockaddr *)&(ctx->addr),
						ctx->addrlen, s_upstream->client_hello->sni, SSL_version(s_stream->ssl), ssl_sess);
				}
			}
			else
			{
				ATOMIC_INC(&(mgr->stat_val[SSL_FAKE_CRT]));
				char* addr_str=tfe_string_addr_create_by_fd(ctx->fd_upstream, CONN_DIR_UPSTREAM);
				TFE_LOG_INFO(mgr->logger, "Fake Cert %s %s : %s", addr_str, s_upstream->client_hello->sni, error_str);
				free(addr_str);
			}
		}
		else
		{
			//Do not perform cert verification on reused session.
			s_upstream->is_server_cert_verify_passed=1;
		}
		if(mgr->log_master_key)
		{
			log_ssl_master_key(s_stream, CONN_DIR_UPSTREAM, mgr->fp_master_key);
		}
		const unsigned char *alpn_data=NULL;
		unsigned int alpn_len=0;
		SSL_get0_alpn_selected(s_stream->ssl, &alpn_data, &alpn_len);
		if(alpn_len>0&&alpn_data!=NULL)
		{
			if(0==memcmp(alpn_data, SSL_ALPN_HTTP_1_1+1, alpn_len))
			{
				s_stream->alpn_selected=SSL_ALPN_HTTP_1_1;
			}
			else if(0==memcmp(alpn_data, SSL_ALPN_HTTP_2+1, alpn_len))
			{
				s_stream->alpn_selected=SSL_ALPN_HTTP_2;
			}
			else
			{
				s_stream->alpn_selected=NULL;
			}
		}
		s_stream->negotiated_version=SSL_version(s_stream->ssl);
		ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_SERVER_SIDE_VERSION, SSL_get_version(s_stream->ssl));
		ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_CERT_VERIFY, s_upstream->is_server_cert_verify_passed);
		promise_success(p, ctx);
	}
	if(s_stream->error)
	{
		ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error));
		snprintf(error_str, sizeof(error_str), "%s, sni=%s", ssl_stream_get_error_string(s_stream->error), s_upstream->client_hello->sni);
		promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str);
	}
	wrap_ssl_connect_server_ctx_free(ctx);
	return;
}

static void peek_chello_on_succ(future_result_t * result, void * user)
{
	struct promise * p = (struct promise *) user;
	struct ssl_connect_server_ctx* ctx = (struct ssl_connect_server_ctx *) promise_get_ctx(p);
	struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id);
	struct ssl_stream* s_stream=NULL;
	struct ssl_chello* chello = ssl_peek_result_release_chello(result);//chello has been saved in ssl_stream.
	if(chello->sni==NULL)
	{
		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_SNI]));
	}
	int ret=0;
	struct ssl_service_status* svc_status=NULL;
	clock_gettime(CLOCK_MONOTONIC, &(ctx->start));
	s_stream= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL, NULL, ctx->tcp_stream);
	svc_status=&s_stream->up_parts.svc_status;
	ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, s_stream->tcp_stream, svc_status);
	if(ret==1)
	{
		TFE_LOG_DEBUG(ctx->mgr->logger, "%s %s service status pinning:%d, mauth:%d, err:%d, ct:%d, ev:%d, ja3_pinning_status:%d",
									s_stream->tcp_stream->str_stream_info,
									chello->sni,
									svc_status->cli_pinning_status,
									svc_status->is_mutual_auth,
									svc_status->has_protocol_errors,
									svc_status->is_ct,
									svc_status->is_ev,
									svc_status->ja3_pinning_status);
	}
    switch (svc_status->ja3_pinning_status)
    {
    case JA3_PINNING_STATUS_NOT_PINNING:
        ctx->mgr->svc_cache->stat.app_not_pinning_cnt++;
        ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_NOT_PINNING);
        break;
    case JA3_PINNING_STATUS_IS_PINNING:
        ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING);
        break;
    case JA3_PINNING_STATUS_UNKNOWN:
		/* fall through */
	default:
        ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_PINNING_STATE, svc_status->cli_pinning_status);
        break;
	}
    
	if(ctx->mgr->on_new_upstream_cb)
	{
		s_stream->up_parts.action=ctx->mgr->on_new_upstream_cb(s_stream, ctx->mgr->upstream_cb_param);
	}
	else
	{
		s_stream->up_parts.action=SSL_ACTION_INTERCEPT;
	}

	if (ATOMIC_READ(&certstore_is_unavailable) > 3)
	{
		s_stream->up_parts.action=SSL_ACTION_PASSTHROUGH;
		ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_PASSTHROUGH_REASON, "Certstore Unavailable");
		TFE_LOG_ERROR(ctx->mgr->logger, "CertStore is unavailable, PASSTHROUGH");
	}
	
	ssl_stream_set_cmsg_uint8(s_stream, TFE_CMSG_SSL_INTERCEPT_STATE, s_stream->up_parts.action);
	ctx->s_stream = s_stream;
	if(s_stream->up_parts.action==SSL_ACTION_PASSTHROUGH)
	{
		promise_dettach_ctx(p);
		promise_success(p, ctx);
		wrap_ssl_connect_server_ctx_free(ctx);
	}
	else
	{
		if (upstream_ossl_init(s_stream))
		{
			promise_dettach_ctx(p);
			promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream ossl init failed");
			wrap_ssl_connect_server_ctx_free(ctx);
			return;
		}
		ctx->bev = bufferevent_openssl_socket_new(evbase, ctx->fd_upstream,
			ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE );

		bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev, 1);
		bufferevent_setcb(ctx->bev, NULL, NULL, ssl_server_connected_eventcb, p);
		bufferevent_enable(ctx->bev, EV_READ | EV_WRITE); //waiting for connect event only

		future_destroy(ctx->f_peek_chello);
		ctx->f_peek_chello = NULL;
	}
	return;
}

static void peek_chello_on_fail(enum e_future_error err, const char * what, void * user)
{
	struct promise * p = (struct promise *) user;
	struct ssl_connect_server_ctx * ctx = (struct ssl_connect_server_ctx *) promise_dettach_ctx(p);
	ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_CHELLO]));
	promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream create failed for no client hello in downstream.");
	wrap_ssl_connect_server_ctx_free(ctx);
	return;
}

void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_socket_t fd_upstream,
	evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream)
{
	struct promise * p = future_to_promise(f);
	struct ssl_connect_server_ctx * ctx = ALLOC(struct ssl_connect_server_ctx, 1);
	int ret = 0;

	ctx->addrlen = sizeof(ctx->addr);
	ret = getpeername(fd_upstream, (struct sockaddr *)&(ctx->addr), &(ctx->addrlen));
	if(ret!=0)
	{
		ssl_connect_server_ctx_free(ctx);
		promise_failed(p, FUTURE_ERROR_EXCEPTION, "upstream fd closed");
		return;
	}
	struct event_base* evbase=tfe_proxy_get_work_thread_evbase(tcp_stream->thread_id);
	ctx->fd_downstream = fd_downstream;
	ctx->fd_upstream = fd_upstream;
	ctx->tcp_stream = tcp_stream;
	ctx->mgr = mgr;
	promise_set_ctx(p, ctx, wrap_ssl_connect_server_ctx_free);

	ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p);
	ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, evbase, mgr->logger);
}


static int ossl_session_ticket_key_callback(SSL *ssl_conn,
    unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx,
    HMAC_CTX *hctx, int enc)
{
    enum STEK_GET_RET ret=STEK_NOT_FOUND;
    const EVP_MD  *digest=EVP_sha256();
    const EVP_CIPHER *cipher=EVP_aes_256_cbc();
    size_t size=32;
	struct ssl_stream* s_stream = (struct ssl_stream*)  SSL_get_ex_data(ssl_conn, SSL_EX_DATA_IDX_SSLSTREAM);
	struct ssl_mgr* mgr= s_stream->mgr;
	void* logger = s_stream->mgr->logger;
	const char * sni = s_stream->peer->up_parts.client_hello->sni;
    struct sess_ticket_box * stek_box = s_stream->mgr->down_stek_box;
	struct sess_ticket_key cur_stek;

    unsigned char buf[33]={0};
    ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY]));
    if (enc == 1)
    {

		sess_ticket_box_get_key_for_enc(stek_box, sni, &cur_stek);
        /* encrypt session stek */
        if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1)
        {
            TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed");
			ret=STEK_ERROR;
			goto leave;
        }
        if (EVP_EncryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1)
        {
            TFE_LOG_ERROR(mgr->logger, "EVP_EncryptInit_ex() failed");
			ret=STEK_ERROR;
			goto leave;
        }
        if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1)
        {
            TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed");
			ret=STEK_ERROR;
			goto leave;
        }
        memcpy(name, cur_stek.name, sizeof(cur_stek.name));
        ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW]));
		ret=STEK_FOUND_FRESH;
    }
    else
    {
       /* decrypt session stek */
		ret=sess_ticket_box_get_key_for_dec(stek_box, sni, name, &cur_stek);

		if(ret==STEK_FOUND_FRESH||ret==STEK_FOUND_STALED)
		{
			if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1)
			{
				TFE_LOG_ERROR(logger, "HMAC_Init_ex() failed");
				ret= STEK_ERROR;
				goto leave;
			}
			if (EVP_DecryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1)
			{
				TFE_LOG_ERROR(logger, "EVP_DecryptInit_ex() failed");
				ret= STEK_ERROR;
				goto leave;
			}
			ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_REUSE]));
		}
		else
		{
			// full handshake and update the session stek_box
			TFE_LOG_INFO(logger, "ssl session stek_box decrypt, key: \"%*s\" not found",(int)(tfe_hexdump(buf, name ,16)-buf), buf);
			ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND]));
		}
    }

leave:
	return (int)ret;
}



/*
 * Called by OpenSSL when a new src SSL session is created.
 * OpenSSL increments the refcount before calling the callback and will
 * decrement it again if we return 0.  Returning 1 will make OpenSSL skip
 * the refcount decrementing.  In other words, return 0 if we did not
 * keep a pointer to the object (which we never do here).
 */
static int ossl_downsess_new_cb(SSL * ssl, SSL_SESSION * sess)
{
	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_CTX_EX_DATA_IDX_SSLMGR);

#ifdef HAVE_SSLV2

    /* Session resumption seems to fail for SSLv2 with protocol
     * parsing errors, so we disable caching for SSLv2. */
    if (SSL_version(ssl) == SSL2_VERSION) {
        return 0;
    }

#endif /* HAVE_SSLV2 */

	if (sess && !mgr->no_sesscache)
	{
		down_session_set(mgr->down_sess_cache, sess);
	}

	return 0;
}

/*
 * Called by OpenSSL when a src SSL session should be removed.
 * OpenSSL calls SSL_SESSION_free() after calling the callback;
 * we do not need to free the reference here.
 */
static void ossl_downsess_remove_cb(SSL_CTX * sslctx, SSL_SESSION * sess)
{
	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_CTX_get_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR);
	assert(mgr != NULL);

	if (sess && !mgr->no_sesscache)
	{
		down_session_del(mgr->down_sess_cache, sess);
	}

	return;
}

/*
 * Called by OpenSSL when a src SSL session is requested by the client.
	OPENSSL_VERSION_NUMBER >= 0x10100000L required.
 */
static SSL_SESSION * ossl_downsess_get_cb(SSL * ssl, const unsigned char * id, int idlen, int * copy)
{
	struct ssl_stream* s_stream = (struct ssl_stream*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM);
	struct ssl_mgr * mgr = s_stream->mgr;
	SSL_SESSION * sess=NULL;
	if(!mgr->no_sesscache)
	{
		*copy = 0;                        /* SSL should not increment reference count of session */
		sess = (SSL_SESSION *) down_session_get(mgr->down_sess_cache, id, idlen);
	}
	return sess;
}

/*
 * Set SSL_CTX options that are the same for incoming and outgoing SSL_CTX.
 */
static void sslctx_set_opts(SSL_CTX * sslctx, struct ssl_mgr * mgr)
{
	SSL_CTX_set_options(sslctx, SSL_OP_ALL);

	SSL_CTX_set_options(sslctx, SSL_OP_TLS_ROLLBACK_BUG);

	SSL_CTX_set_options(sslctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION);

	SSL_CTX_set_options(sslctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS);
	if (mgr->no_ssl2)
	{
		SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2);
	}
    if (mgr->no_ssl3)
	{
        SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv3);
    }

	if (mgr->no_tls10)
	{
		SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1);
	}
	if (mgr->no_tls11)
	{
		SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_1);
	}
	if (mgr->no_tls12)
	{
		SSL_CTX_set_options(sslctx, SSL_OP_NO_TLSv1_2);
	}
	if(mgr->no_sessticket)
	{
		SSL_CTX_set_options(sslctx, SSL_OP_NO_TICKET);
	}
	if (!mgr->sslcomp)
	{
		SSL_CTX_set_options(sslctx, SSL_OP_NO_COMPRESSION);
	}
}


static int alpn_select_proto_cb(SSL *ssl, const unsigned char **out,
                                unsigned char *outlen, const unsigned char *in,
                                unsigned int inlen, void *arg) {
	int rv;
	const unsigned char* selected_alpn=(const unsigned char*)arg;
	(void)ssl;
	unsigned char* _out=NULL;
	unsigned char _outlen=0;
	rv=SSL_select_next_proto(&_out, &_outlen, in, inlen, selected_alpn, strlen((const char*)selected_alpn));
	if(rv==OPENSSL_NPN_NEGOTIATED)
	{
		*out=_out;
		*outlen=_outlen;
		return SSL_TLSEXT_ERR_OK;
	}
	else //rv==OPENSSL_NPN_NO_OVERLAP
	{
		return SSL_TLSEXT_ERR_NOACK;
	}
}


/*
 * Create and set up a new SSL_CTX instance for terminating SSL.
 * Set up all the necessary callbacks, the keyring, the keyring chain and key.
 */
void downstream_ossl_init(struct ssl_stream *s_stream)
{
	struct ssl_mgr * mgr=s_stream->mgr;
	SSL_CTX * sslctx = SSL_CTX_new(mgr->sslmethod());
	if (!sslctx) return;
	struct keyring * crt=s_stream->down_parts.keyring;
	int negotiated_version=s_stream->peer->negotiated_version;
	const unsigned char* selected_alpn=s_stream->peer->alpn_selected;

	SSL * ssl = NULL;
	UNUSED int ret = 0;
	sslctx_set_opts(sslctx, mgr);
	SSL_CTX_set_cipher_list(sslctx, mgr->default_ciphers);

	//TFE's OPENSSL_VERSION_NUMBER >= 0x10100000L
	if(negotiated_version)
	{
		if (SSL_CTX_set_min_proto_version(sslctx, negotiated_version) == 0 ||
			SSL_CTX_set_max_proto_version(sslctx, negotiated_version) == 0)
		{
			SSL_CTX_free(sslctx);
			return;
		}
	}
	else if(mgr->ssl_min_version)
	{
		if (SSL_CTX_set_min_proto_version(sslctx, mgr->ssl_min_version) == 0 ||
			SSL_CTX_set_max_proto_version(sslctx, mgr->ssl_max_version) == 0)
		{
			SSL_CTX_free(sslctx);
			return;
		}
	}

	SSL_CTX_sess_set_new_cb(sslctx, ossl_downsess_new_cb);
	SSL_CTX_sess_set_remove_cb(sslctx, ossl_downsess_remove_cb);
	SSL_CTX_sess_set_get_cb(sslctx, ossl_downsess_get_cb);
	if(!mgr->no_sessticket&&s_stream->peer->up_parts.client_hello->sni)
	{
		SSL_CTX_set_tlsext_ticket_key_cb(sslctx, ossl_session_ticket_key_callback);
	}
	SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL);
	SSL_CTX_set_session_id_context(sslctx, (const unsigned char *) mgr->ssl_session_context,
		sizeof(mgr->ssl_session_context));
	ret = SSL_CTX_set_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR, mgr);//used by down session cache get
	assert(ret == 1);
	if (mgr->dh)
	{
		SSL_CTX_set_tmp_dh(sslctx, mgr->dh);
	}
	else
	{
		SSL_CTX_set_tmp_dh_callback(sslctx, ssl_tmp_dh_callback);
	}

	if (mgr->ecdhcurve)
	{
		EC_KEY * ecdh = ssl_ec_by_name(mgr->ecdhcurve);
		SSL_CTX_set_tmp_ecdh(sslctx, ecdh);
		EC_KEY_free(ecdh);
	}
	else
	{
		EC_KEY * ecdh = ssl_ec_by_name(NULL);
		SSL_CTX_set_tmp_ecdh(sslctx, ecdh);
		EC_KEY_free(ecdh);
	}
	if(s_stream->peer->up_parts.apln_enabled && selected_alpn)
	{
		SSL_CTX_set_alpn_select_cb(sslctx, alpn_select_proto_cb, (void*)selected_alpn);
	}
	SSL_CTX_use_certificate(sslctx, crt->cert);
	SSL_CTX_use_PrivateKey(sslctx, crt->key);

	for (int i = 0; i < sk_X509_num(crt->chain); i++)
	{
		X509 * c = sk_X509_value(crt->chain, i);
		ssl_x509_refcount_inc(c);                    /* next call consumes a reference */
		SSL_CTX_add_extra_chain_cert(sslctx, c);
	}

	ssl = SSL_new(sslctx);
	SSL_CTX_free(sslctx);            // SSL_new() increments refcount
	sslctx = NULL;

	ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream);
	assert(ret == 1);

	if (mgr->ssl_mode_release_buffers == 1)
	{
		/* lower memory footprint for idle connections */
		SSL_set_mode(ssl, SSL_get_mode(ssl) | SSL_MODE_RELEASE_BUFFERS);
	}
	s_stream->ssl=ssl;
	return;
}

void ssl_connect_client_ctx_free(struct ssl_connect_client_ctx * ctx)
{
	X509_free(ctx->origin_crt);
	ctx->origin_crt=NULL;

	if (ctx->f_ask_keyring != NULL)
	{
		future_destroy(ctx->f_ask_keyring);
		ctx->f_ask_keyring = NULL;
	}
	//on success, bev_down and downstream has been transfered to caller by release**
	if (ctx->bev_down != NULL)
	{
		bufferevent_free(ctx->bev_down);
	}

	if (ctx->downstream != NULL)
	{
		ssl_stream_free(ctx->downstream);
	}
	free(ctx);
	return;
}

void wrap_ssl_connect_client_ctx_free(void * p)
{
	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *)p;
	ssl_connect_client_ctx_free(ctx);
}

struct ssl_stream * ssl_downstream_create_result_release_stream(future_result_t * result)
{
	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result;
	struct ssl_stream * ret = ctx->downstream;
	ctx->downstream = NULL;
	return ret;
}
struct bufferevent * ssl_downstream_create_result_release_bev(future_result_t * result)
{
	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) result;
	struct bufferevent * ret = ctx->bev_down;
	ctx->bev_down = NULL;
	return ret;
}

static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, void * arg)
{
	struct promise * p = (struct promise *) arg;
	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p);

	struct ssl_stream * s_stream = ctx->downstream;
	struct ssl_upstream_parts* s_upstream= &(ctx->peer->up_parts);
	struct ssl_mgr* mgr=s_stream->mgr;
	char error_str[TFE_STRING_MAX]={0};
	uint64_t jiffies_ms=0;
	unsigned long sslerr=0;
	if (events & BEV_EVENT_ERROR)
	{
		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
		sslerr=ssl_stream_log_error(bev, CONN_DIR_DOWNSTREAM, mgr);
		if(sslerr)
		{
			ssl_stream_process_error(s_stream, sslerr, mgr);
		}
		s_stream->error=SSL_STREAM_R_CLIENT_PROTOCOL_ERROR;
	}
	else if(events & BEV_EVENT_EOF)
	{
		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
		ssl_stream_process_zero_eof(s_stream, mgr);
	}
	else if(events & BEV_EVENT_TIMEOUT)
	{
		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
		s_stream->error=SSL_STREAM_R_CONNECT_CLIENT_TIMEOUT;
	}
	else if(events & BEV_EVENT_CONNECTED)
	{

		clock_gettime(CLOCK_MONOTONIC, &(ctx->end));
		jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000;
		if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS)
		{
			TFE_LOG_INFO(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s",
										jiffies_ms,
										s_stream->tcp_stream->str_stream_info,
										s_upstream->client_hello->sni);
		}
		s_stream->connect_latency_ms=jiffies_ms;
		ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_CLIENT_SIDE_LATENCY, jiffies_ms);
		bufferevent_disable(ctx->bev_down, EV_READ | EV_WRITE);
		bufferevent_setcb(ctx->bev_down, NULL, NULL, NULL, NULL); //leave a clean bev for on_success

		if(mgr->log_master_key)
		{
			log_ssl_master_key(s_stream, CONN_DIR_DOWNSTREAM, mgr->fp_master_key);
		}
		s_stream->negotiated_version=SSL_version(s_stream->ssl);
		ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_CLIENT_SIDE_VERSION, SSL_get_version(s_stream->ssl));
		promise_success(p, ctx);
	}

	if(s_stream->error)
	{
		ssl_stream_set_cmsg_string(s_stream, TFE_CMSG_SSL_ERROR, ssl_stream_get_error_string(s_stream->error));
		snprintf(error_str, sizeof(error_str), "%s : sni=%s", ssl_stream_get_error_string(s_stream->error), s_upstream->client_hello->sni);
		promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str);
	}

	ssl_connect_client_ctx_free(ctx);
	return;
}

void ask_keyring_on_succ(void * result, void * user)
{
	struct promise * p = (struct promise *) user;
	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_get_ctx(p);

	struct keyring * kyr = NULL;
	struct ssl_mgr * mgr = ctx->ssl_mgr;
	struct event_base* evbase=tfe_proxy_get_work_thread_evbase(ctx->tcp_stream->thread_id);

	kyr = key_keeper_release_keyring(result);	//kyr will be freed at ssl downstream closing.

	clock_gettime(CLOCK_MONOTONIC, &(ctx->start));
	ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL,
										kyr, ctx->peer, ctx->tcp_stream);
	downstream_ossl_init(ctx->downstream);
	ctx->bev_down = bufferevent_openssl_socket_new(evbase, ctx->fd_downstream, ctx->downstream->ssl,
		BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE);
	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1);

	bufferevent_setcb(ctx->bev_down, NULL, NULL, ssl_client_connected_eventcb, p);
	bufferevent_enable(ctx->bev_down, EV_READ | EV_WRITE); //waiting for connect event only

	future_destroy(ctx->f_ask_keyring);
	ctx->f_ask_keyring = NULL;

}

void ask_keyring_on_fail(enum e_future_error error, const char * what, void * user)
{
	struct promise * p = (struct promise *) user;
	struct ssl_connect_client_ctx * ctx = (struct ssl_connect_client_ctx *) promise_dettach_ctx(p);
	promise_failed(p, error, what);
	ssl_connect_client_ctx_free(ctx);
	return;
}

/*
 * Create a SSL stream for the incoming connection, based on the upstream.
 */
void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct ssl_stream * upstream,
	evutil_socket_t fd_downstream, struct tfe_stream* tcp_stream)
{

	assert(upstream->dir == CONN_DIR_UPSTREAM);
	const char* sni=NULL;
 	struct ssl_connect_client_ctx * ctx = ALLOC(struct ssl_connect_client_ctx, 1);
	ctx->ssl_mgr = mgr;
	ctx->fd_downstream = fd_downstream;
	ctx->tcp_stream = tcp_stream;
	struct event_base * evbase=tfe_proxy_get_work_thread_evbase(tcp_stream->thread_id);
	struct evdns_base* dnsbase=tfe_proxy_get_work_thread_dnsbase(tcp_stream->thread_id);
	struct evhttp_connection *evhttp=tfe_proxy_get_work_thread_evhttp(tcp_stream->thread_id);

	if (upstream != NULL)
	{
		ctx->peer = upstream;
		ctx->origin_crt = SSL_get_peer_certificate(upstream->ssl);
		sni=upstream->up_parts.client_hello->sni;
	}

	struct promise * p = future_to_promise(f);
	promise_set_ctx(p, ctx, wrap_ssl_connect_client_ctx_free);

	ctx->f_ask_keyring = future_create("ask_kyr",ask_keyring_on_succ, ask_keyring_on_fail, p);
	ctx->is_origin_crt_verify_passed = upstream->up_parts.is_server_cert_verify_passed;
	int keyring_id = 0;
	if (ctx->is_origin_crt_verify_passed)
	{
		keyring_id = upstream->up_parts.keyring_for_trusted;
	}
	else
	{
		keyring_id = upstream->up_parts.keyring_for_untrusted;
	}
	key_keeper_async_ask(ctx->f_ask_keyring, mgr->key_keeper, sni, keyring_id, ctx->origin_crt, ctx->is_origin_crt_verify_passed,
		evbase, dnsbase, evhttp);
	return;
}

/*
 * Cleanly shutdown an SSL session on file descriptor fd using low-level
 * file descriptor readiness events on event base evbase.
 * Guarantees that SSL and the corresponding SSL_CTX are freed and the
 * socket is closed, eventually, or in the case of fatal errors, immediately.
 */
void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, struct bufferevent * bev)
{
	UNUSED struct ssl_shutdown_ctx * sslshutctx = NULL;
	evutil_socket_t fd=-1;
	fd=bufferevent_getfd(bev);
	assert(fd==s_stream->_do_not_use.fd);
	unsigned long sslerr=0;

	if (s_stream->dir == CONN_DIR_UPSTREAM)
	{
		size_t rx_offset_this_time = 0;
		tfe_stream_info_get(s_stream->tcp_stream, INFO_FROM_UPSTREAM_RX_OFFSET, &rx_offset_this_time, sizeof(rx_offset_this_time));
		const char * sni = (s_stream->up_parts.client_hello && s_stream->up_parts.client_hello->sni) ? s_stream->up_parts.client_hello->sni : "null";
		TFE_LOG_DEBUG(g_default_logger, "ssl up stream close, rx_offset:%zu, sni:%s", rx_offset_this_time, sni);
	}

	if(errno)
	{
		sslerr=ssl_stream_log_error(bev, s_stream->dir, s_stream->mgr);
		if(sslerr)
		{
			ssl_stream_process_error(s_stream, sslerr, s_stream->mgr);
		}
	}
	//sslshutctx = ssl_shutdown_ctx_new(s_stream, evbase);
	//pxy_ssl_shutdown_cb(fd, 0, sslshutctx);
	bufferevent_setcb(bev, NULL, NULL, NULL, NULL);
	SSL_set_shutdown(s_stream->ssl, SSL_RECEIVED_SHUTDOWN);
	SSL_shutdown(s_stream->ssl);
	bufferevent_disable(bev, EV_READ|EV_WRITE);
	struct bufferevent *ubev = bufferevent_get_underlying(bev);
	if (ubev) {
		bufferevent_disable(ubev, EV_READ|EV_WRITE);
		bufferevent_setfd(ubev, -1);
		bufferevent_setcb(ubev, NULL, NULL, NULL, NULL);
		bufferevent_free(ubev);
	}

	ssl_stream_free(s_stream);
}

int ssl_manager_add_trust_ca(struct ssl_mgr* mgr, const char* pem_file)
{
	int ret = ssl_trusted_cert_storage_add(mgr->trust_CA_store, SSL_X509_OBJ_CERT, pem_file);
	if (ret == 1)
	{
        ATOMIC_INC(&((*(mgr->svc_cache)).stat.trusted_cert_cnt));
    }
	return ret;
}
int ssl_manager_del_trust_ca(struct ssl_mgr* mgr, const char* pem_file)
{
	int ret = ssl_trusted_cert_storage_del(mgr->trust_CA_store, SSL_X509_OBJ_CERT, pem_file);
	if (ret == 1)
	{
        ATOMIC_DEC(&((*(mgr->svc_cache)).stat.trusted_cert_cnt));
    }
	return ret;
}
int ssl_manager_add_crl(struct ssl_mgr* mgr, const char* pem_file)
{
	return ssl_trusted_cert_storage_add(mgr->trust_CA_store, SSL_X509_OBJ_CRL, pem_file);
}
int ssl_manager_del_crl(struct ssl_mgr* mgr, const char* pem_file)
{
	return ssl_trusted_cert_storage_del(mgr->trust_CA_store, SSL_X509_OBJ_CRL, pem_file);
}
void ssl_manager_reset_trust_ca(struct ssl_mgr* mgr)
{
	ATOMIC_ZERO(&((*(mgr->svc_cache)).stat.trusted_cert_cnt));
	ssl_trusted_cert_storage_reset(mgr->trust_CA_store);
	return;
}
void ssl_manager_reset_trust_ca_finish(struct ssl_mgr* mgr)
{
    ssl_trusted_cert_storage_reset_finish(mgr->trust_CA_store);
    return;
}
int ssl_stream_set_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT  opt_type, int opt_val)
{
	struct cert_verify_param *verify_param=&(upstream->up_parts.verify_param);
	assert(upstream->dir==CONN_DIR_UPSTREAM);
	switch(opt_type)
	{
		case	SSL_STREAM_OPT_NO_VERIFY_SELF_SIGNED:
			verify_param->no_verify_self_signed=opt_val;
			break;
		case	SSL_STREAM_OPT_NO_VERIFY_COMMON_NAME:
			verify_param->no_verify_cn=opt_val;
			break;
		case	SSL_STREAM_OPT_NO_VERIFY_ISSUER:
			verify_param->no_verify_issuer=opt_val;
			break;
		case	SSL_STREAM_OPT_NO_VERIFY_EXPIRY_DATE:
			verify_param->no_verify_expiry_date=opt_val;
			break;
		case	SSL_STREAM_OPT_BLOCK_FAKE_CERT:
			upstream->up_parts.block_fake_cert=opt_val;
			break;
		case	SSL_STREAM_OPT_PROTOCOL_MIN_VERSION:
			upstream->ssl_min_version=opt_val;
			break;
		case	SSL_STREAM_OPT_PROTOCOL_MAX_VERSION:
			upstream->ssl_max_version=opt_val;
			break;
		case	SSL_STREAM_OPT_ENABLE_ALPN:
			upstream->up_parts.apln_enabled=opt_val;
			break;
		case	SSL_STREAM_OPT_KEYRING_FOR_TRUSTED:
			upstream->up_parts.keyring_for_trusted=opt_val;
			break;
		case	SSL_STREAM_OPT_KEYRING_FOR_UNTRUSTED:
			upstream->up_parts.keyring_for_untrusted=opt_val;
			break;
		default:
			assert(0);
			return 0;

	}
	return 1;
}
int ssl_stream_get_integer_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, int *opt_val)
{
	struct ssl_service_status* svc=&upstream->up_parts.svc_status;
	UNUSED int ret=0;
	switch(opt_type)
	{
		case	SSL_STREAM_OPT_IS_EV_CERT:
			*opt_val=svc->is_ev;
			break;
		case	SSL_STREAM_OPT_IS_CT_CERT:
			*opt_val=svc->is_ct;
			break;
		case	SSL_STREAM_OPT_IS_MUTUAL_AUTH:
			*opt_val=svc->is_mutual_auth;
			break;
		case	SSL_STREAM_OPT_PINNING_STATUS:
			*opt_val=svc->cli_pinning_status;
			break;
		case	SSL_STREAM_OPT_JA3_PINNING_STATUS:
            *opt_val=svc->ja3_pinning_status;
			break;
		case	SSL_STREAM_OPT_HAS_PROTOCOL_ERRORS:
			*opt_val=svc->has_protocol_errors;
			break;
		default:
			return -1;
	}
	return 0;

}

uint64_t ssl_stream_get_policy_id(struct ssl_stream *upstream)
{
	uint16_t out_size;
	uint64_t policy_id = 0;
	struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream);
	int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_POLICY_ID, (unsigned char *)&policy_id, sizeof(policy_id), &out_size);
	assert(ret == 0);

	return policy_id;
}

int ssl_stream_get_decrypted_profile_id(struct ssl_stream *upstream)
{
	uint16_t out_size;
	int profile_id = 0;
	struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream);
	int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DECRYPTION_PROFILE_ID, (unsigned char *)&profile_id, sizeof(profile_id), &out_size);
	assert(ret == 0);

	return profile_id;
}

int ssl_stream_get_trusted_keyring_profile_id(struct ssl_stream *upstream)
{
	uint16_t out_size;
	int keyring_id = 0;
	struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream);
	int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_KEYRING_FOR_TRUSTED_ID, (unsigned char *)&keyring_id, sizeof(keyring_id), &out_size);
	assert(ret == 0);

	return keyring_id;
}

int ssl_stream_get_untrusted_keyring_profile_id(struct ssl_stream *upstream)
{
	uint16_t out_size;
	int keyring_id = 0;
	struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(upstream->tcp_stream);
	int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_KEYRING_FOR_UNTRUSTED, (unsigned char *)&keyring_id, sizeof(keyring_id), &out_size);
	assert(ret == 0);

	return keyring_id;
}

int ssl_stream_get_string_opt(struct ssl_stream *upstream, enum SSL_STREAM_OPT opt_type, char* in_buff, size_t sz)
{
	const char* sni=upstream->up_parts.client_hello->sni?upstream->up_parts.client_hello->sni:"null";
	switch(opt_type)
	{
		case SSL_STREAM_OPT_SNI:
			strncpy(in_buff, sni, sz);
			break;
		case SSL_STREAM_OPT_ADDR:
			strncpy(in_buff, upstream->tcp_stream->str_stream_info, sz);
			break;
		default:
			assert(0);
			return -1;
	}
	return 0;
}